diff --git a/.github/workflows/autotest.yml b/.github/workflows/autotest.yml index 43a0e6d464..df6dd96ca9 100644 --- a/.github/workflows/autotest.yml +++ b/.github/workflows/autotest.yml @@ -178,19 +178,19 @@ jobs: - name: Set repository variable id: repo run: echo "repo=$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT - - name: Build Python 3.9 image locally - if: needs.runtime_images.outputs.build_locally == 'true' && matrix.cfg.PYTHON == '3.9' + - name: Build Python 3.11 image locally + if: needs.runtime_images.outputs.build_locally == 'true' && matrix.cfg.PYTHON == '3.11' uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: . file: etc/docker/test/runtime.Dockerfile target: final build-args: | - PYTHON=3.9 + PYTHON=3.11 push: false - tags: ${{ needs.runtime_images.outputs.py39_image }} + tags: ${{ needs.runtime_images.outputs.py311_image }} outputs: type=docker - cache-from: type=registry,ref=ghcr.io/${{ steps.repo.outputs.repo }}/rucio-dev-runtime:py39-buildcache + cache-from: type=registry,ref=ghcr.io/${{ steps.repo.outputs.repo }}/rucio-dev-runtime:py311-buildcache continue-on-error: false - name: Build Python 3.10 image locally if: needs.runtime_images.outputs.build_locally == 'true' && matrix.cfg.PYTHON == '3.10' @@ -208,8 +208,8 @@ jobs: continue-on-error: false - name: Run test with cfg env: - PY39_RUNTIME_IMAGE: ${{ needs.runtime_images.outputs.py39_image }} + PY311_RUNTIME_IMAGE: ${{ needs.runtime_images.outputs.py311_image }} PY310_RUNTIME_IMAGE: ${{ needs.runtime_images.outputs.py310_image }} BUILD_LOCALLY: ${{ needs.runtime_images.outputs.build_locally }} run: | - echo '{"matrix": ${{ toJson(matrix.cfg) }}, "runtime_images": {"3.9": "${{ needs.runtime_images.outputs.py39_image }}", "3.10": "${{ needs.runtime_images.outputs.py310_image }}"}, "build_locally": "${{ needs.runtime_images.outputs.build_locally }}" }' | ./tools/test/run_tests.py + echo '{"matrix": ${{ toJson(matrix.cfg) }}, "runtime_images": {"3.11": "${{ needs.runtime_images.outputs.py311_image }}", "3.10": "${{ needs.runtime_images.outputs.py310_image }}"}, "build_locally": "${{ needs.runtime_images.outputs.build_locally }}" }' | ./tools/test/run_tests.py diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 055ebf39e2..02ba935a1c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -12,7 +12,7 @@ concurrency: jobs: runtime_images: - name: Build Python 3.9 Runtime Image + name: Build Python 3.10 Runtime Image uses: ./.github/workflows/runtime_images.yml permissions: contents: read @@ -121,7 +121,7 @@ jobs: build-args: | PYTHON=3.9 push: false - tags: ${{ needs.runtime_images.outputs.py39_image }} + tags: ${{ needs.runtime_images.outputs.py310_image }} outputs: type=docker continue-on-error: false - name: Pull all required images @@ -133,7 +133,7 @@ jobs: shell: bash run: | docker image ls - sed -i 's;image: .*rucio-dev.*;image: ${{ needs.runtime_images.outputs.py39_image }};' \ + sed -i 's;image: .*rucio-dev.*;image: ${{ needs.runtime_images.outputs.py310_image }};' \ $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml - name: Start containers run: | diff --git a/.github/workflows/runtime_images.yml b/.github/workflows/runtime_images.yml index 2caa9c64f2..8d68d4076c 100644 --- a/.github/workflows/runtime_images.yml +++ b/.github/workflows/runtime_images.yml @@ -3,9 +3,9 @@ name: Build Runtime Images on: workflow_call: outputs: - py39_image: - description: "Python 3.9 runtime image tag" - value: ${{ jobs.build_runtime_images.outputs.py39_image }} + py311_image: + description: "Python 3.11 runtime image tag" + value: ${{ jobs.build_runtime_images.outputs.py311_image }} py310_image: description: "Python 3.10 runtime image tag" value: ${{ jobs.build_runtime_images.outputs.py310_image }} @@ -67,14 +67,14 @@ jobs: COMMIT_REF="${{ steps.metadata.outputs.commit_ref }}" REPO=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - for PYVER in 3.9 3.10; do + for PYVER in 3.11 3.10; do HASH=$(echo "PYTHON=${PYVER}" | cat - $FILES_TO_HASH | sha256sum | cut -d' ' -f1 | head -c 12) IMAGE="ghcr.io/${REPO}/rucio-dev-runtime:py${PYVER//.}-${HASH}" echo "Python ${PYVER} image tag: $IMAGE" # Export dynamic outputs - if [[ "$PYVER" == "3.9" ]]; then - echo "py39_image=$IMAGE" >> $GITHUB_OUTPUT + if [[ "$PYVER" == "3.11" ]]; then + echo "py311_image=$IMAGE" >> $GITHUB_OUTPUT else echo "py310_image=$IMAGE" >> $GITHUB_OUTPUT fi diff --git a/etc/docker/test/extra/rucio_autotests_common.cfg b/etc/docker/test/extra/rucio_autotests_common.cfg index 5fb0a3cdd4..fbcc721e6a 100644 --- a/etc/docker/test/extra/rucio_autotests_common.cfg +++ b/etc/docker/test/extra/rucio_autotests_common.cfg @@ -140,7 +140,7 @@ email_from = Rucio email_test = spamspamspam@cern.ch [policy] -package = atlas_rucio_policy_package +#package = atlas_rucio_policy_package #permission = atlas #schema = atlas lfn2pfn_algorithm_default = hash diff --git a/etc/docker/test/extra/rucio_default.cfg b/etc/docker/test/extra/rucio_default.cfg index e180f1023f..e47de4dfff 100644 --- a/etc/docker/test/extra/rucio_default.cfg +++ b/etc/docker/test/extra/rucio_default.cfg @@ -160,7 +160,7 @@ email_from = Rucio email_test = spamspamspam@cern.ch [policy] -package = atlas_rucio_policy_package +#package = atlas_rucio_policy_package #permission = atlas #schema = atlas lfn2pfn_algorithm_default = hash diff --git a/etc/docker/test/matrix.yml b/etc/docker/test/matrix.yml index 737fdc64e3..7a9c4a2b45 100644 --- a/etc/docker/test/matrix.yml +++ b/etc/docker/test/matrix.yml @@ -2,10 +2,10 @@ dists: - id: alma9 allow: python: - - "3.9" + - "3.11" - "3.10" python: - - id: "3.9" + - id: "3.11" allow: suites: - remote_dbs diff --git a/etc/docker/test/matrix_nightly.yml b/etc/docker/test/matrix_nightly.yml index 37448d6efd..9e887c07ad 100644 --- a/etc/docker/test/matrix_nightly.yml +++ b/etc/docker/test/matrix_nightly.yml @@ -2,10 +2,10 @@ dists: - id: alma9 allow: python: - - "3.9" + - "3.10" python: - - "3.9" - "3.10" + - "3.11" suites: - id: client RDBMS: postgres14 diff --git a/etc/docker/test/runtime.Dockerfile b/etc/docker/test/runtime.Dockerfile index 8a26ec92bd..07fb1781e7 100644 --- a/etc/docker/test/runtime.Dockerfile +++ b/etc/docker/test/runtime.Dockerfile @@ -9,6 +9,7 @@ FROM almalinux:9.1 AS base ENV PYTHON_VENV="/opt/venv" ENV PATH="${PYTHON_VENV}/bin:${PATH}" ENV PYTHON_310_PATCH_VERSION="4" + ENV PYTHON_311_PATCH_VERSION="11" ENV RUCIO_HOME="/opt/rucio" FROM base AS oracle-client @@ -17,17 +18,14 @@ FROM base AS oracle-client echo "/usr/lib/oracle/19.12/client64/lib" > /etc/ld.so.conf.d/oracle-instantclient.conf; FROM base AS python - RUN if [ "$PYTHON" == "3.9" ] ; then \ - dnf install -y epel-release.noarch && \ - dnf install -y 'dnf-command(config-manager)' && \ - dnf config-manager --set-enabled crb && \ - dnf -y update && \ - dnf -y install boost-python3 python3-pip python3-devel && \ - dnf remove --assumeyes python3-setuptools && \ - python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install --upgrade setuptools wheel; \ - elif [ "$PYTHON" == "3.10" ] ; then \ + RUN if [ "$PYTHON" == "3.10" ] ; then \ PYTHON_VERSION="3.10.${PYTHON_310_PATCH_VERSION}" && \ + COMPILE_FROM_SOURCE=true; \ + elif [ "$PYTHON" == "3.11" ] ; then \ + PYTHON_VERSION="3.11.${PYTHON_311_PATCH_VERSION}" && \ + COMPILE_FROM_SOURCE=true; \ + fi && \ + if [ "$COMPILE_FROM_SOURCE" = "true" ] ; then \ dnf install -y 'dnf-command(config-manager)' && \ dnf config-manager --enable crb && \ dnf -y update && \ @@ -57,11 +55,11 @@ FROM python AS gfal2 if [ "$PYTHON" == "3.9" ] ; then \ dnf -y install gfal2-python3 && \ cp /usr/lib64/python3.9/site-packages/gfal2.so /usr/lib64/gfal2.so; \ - elif [ "$PYTHON" == "3.10" ] ; then \ + elif [ "$PYTHON" == "3.10" ] || [ "$PYTHON" == "3.11" ] ; then \ wget https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz && \ tar -xvzf boost_1_80_0.tar.gz && \ cd boost_1_80_0 && \ - ./bootstrap.sh --with-libraries=python --with-python=/usr/bin/python3.10 --prefix=/usr --libdir=/usr/local/lib && \ + ./bootstrap.sh --with-libraries=python --with-python=/usr/bin/python${PYTHON} --prefix=/usr --libdir=/usr/local/lib && \ ./b2 --with-python --libdir=/usr/local/lib --link=shared && \ cp /usr/local/src/boost_1_80_0/stage/lib/lib* /usr/lib64/ && \ dnf install -y git dnf-plugins-core git rpm-build tree which cmake make gcc gcc-c++ && \ @@ -81,7 +79,7 @@ FROM python AS mod_wsgi RUN if [ "$PYTHON" == "3.9" ] ; then \ dnf install -y python3-mod_wsgi && \ cp /usr/lib64/httpd/modules/mod_wsgi_python3.so /usr/lib64/httpd/modules/mod_wsgi.so; \ - elif [ "$PYTHON" == "3.10" ] ; then \ + elif [ "$PYTHON" == "3.10" ] || [ "$PYTHON" == "3.11" ] ; then \ dnf install -y httpd-devel && \ curl -sSL https://github.com/GrahamDumpleton/mod_wsgi/archive/4.9.1.tar.gz | tar xzv && \ cd mod_wsgi-4.9.1 && \ diff --git a/lib/rucio/alembicrevision.py b/lib/rucio/alembicrevision.py index 8221a49f4c..1767b32c73 100644 --- a/lib/rucio/alembicrevision.py +++ b/lib/rucio/alembicrevision.py @@ -12,4 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -ALEMBIC_REVISION = '3b943000da18' # the current alembic head revision +ALEMBIC_REVISION = '8ab4d628cffb' # the current alembic head revision +# ALEMBIC_REVISION = '3b943000da18' # the previous alembic head revision diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py index 3ca6a06886..37425d0aa0 100644 --- a/lib/rucio/client/uploadclient.py +++ b/lib/rucio/client/uploadclient.py @@ -727,14 +727,15 @@ def _register_file( file_did = {'scope': file_scope, 'name': file_name} replica_for_api = self._convert_file_for_api(file) try: - # if the remote checksum is different, this DID must not be used + # if the remote checksum is different, this DID must not be used. Use checksum column (not the legacy columns) to detect this + # and raise an exception meta = self.client.get_metadata(file_scope, file_name) logger(logging.INFO, 'File DID already exists') - logger(logging.DEBUG, 'local checksum: %s, remote checksum: %s' % (file['adler32'], meta['adler32'])) + logger(logging.DEBUG, 'local checksum: %s, remote checksum: %s' % (file['checksum']['adler32'], meta['checksum']['adler32'])) - if str(meta['adler32']).lstrip('0') != str(file['adler32']).lstrip('0'): + if str(meta['checksum']['adler32']).lstrip('0') != str(file['checksum']['adler32']).lstrip('0'): logger(logging.ERROR, - 'Local checksum %s does not match remote checksum %s' % (file['adler32'], meta['adler32'])) + 'Local checksum %s does not match remote checksum %s' % (file['checksum']['adler32'], meta['adler32'])) raise DataIdentifierAlreadyExists # add the file to rse if it is not registered yet @@ -845,8 +846,7 @@ def _collect_file_info( new_item['basename'] = os.path.basename(filepath) new_item['bytes'] = os.stat(filepath).st_size - new_item['adler32'] = adler32(filepath) - new_item['md5'] = md5(filepath) + new_item['checksum'] = {'md5': md5(filepath), 'adler32': adler32(filepath)} new_item['meta'] = {'guid': self._get_file_guid(new_item)} new_item['state'] = 'C' if not new_item.get('did_scope'): @@ -975,8 +975,9 @@ def _convert_file_for_api( replica['scope'] = file['did_scope'] replica['name'] = file['did_name'] replica['bytes'] = file['bytes'] - replica['adler32'] = file['adler32'] - replica['md5'] = file['md5'] + replica['adler32'] = file['checksum'].get('adler32') + replica['md5'] = file['checksum'].get('md5') + replica['checksum'] = file['checksum'] replica['meta'] = file['meta'] replica['state'] = file['state'] pfn = file.get('pfn') diff --git a/lib/rucio/common/schema/generic.py b/lib/rucio/common/schema/generic.py index 6bb2a14dc5..b1466e1f98 100644 --- a/lib/rucio/common/schema/generic.py +++ b/lib/rucio/common/schema/generic.py @@ -129,6 +129,11 @@ "type": "string", "pattern": "^[a-fA-F\\d]{8}$"} +CHECKSUM = {"description": "Checksum dictionary", + "type": "object", + "properties": {"adler32": {"type": "string"}, + "md5": {"type": "string"}}} + WEIGHT = {"description": "Rule weight", "type": ["string", "null"]} @@ -254,6 +259,7 @@ "bytes": BYTES, "adler32": ADLER32, "md5": MD5, + "checksum": CHECKSUM, "state": REPLICA_STATE, "pfn": PFN}, "required": ["scope", "name"], @@ -273,6 +279,7 @@ "bytes": BYTES, "adler32": ADLER32, "md5": MD5, + "checksum": CHECKSUM, "state": REPLICA_STATE, "pfn": PFN}, "required": ["scope", "name"], diff --git a/lib/rucio/common/schema/generic_multi_vo.py b/lib/rucio/common/schema/generic_multi_vo.py index 7e9c3e0048..ab5b6a1518 100644 --- a/lib/rucio/common/schema/generic_multi_vo.py +++ b/lib/rucio/common/schema/generic_multi_vo.py @@ -130,6 +130,11 @@ "type": "string", "pattern": "^[a-fA-F\\d]{8}$"} +CHECKSUM = {"description": "Checksum dictionary", + "type": "object", + "properties": {"adler32": {"type": "string"}, + "md5": {"type": "string"}}} + WEIGHT = {"description": "Rule weight", "type": ["string", "null"]} @@ -228,6 +233,7 @@ "bytes": BYTES, "adler32": ADLER32, "md5": MD5, + "checksum": CHECKSUM, "state": REPLICA_STATE, "pfn": PFN}, "required": ["scope", "name"], @@ -247,6 +253,7 @@ "bytes": BYTES, "adler32": ADLER32, "md5": MD5, + "checksum": CHECKSUM, "state": REPLICA_STATE, "pfn": PFN}, "required": ["scope", "name"], diff --git a/lib/rucio/common/types.py b/lib/rucio/common/types.py index 15160bf415..cbb35d0ca8 100644 --- a/lib/rucio/common/types.py +++ b/lib/rucio/common/types.py @@ -414,6 +414,7 @@ class FileToUploadWithCollectedInfoDict(FileToUploadDict): dirname: str upload_result: dict bytes: int + checksum: dict[str, Any] class FileToUploadWithCollectedAndDatasetInfoDict(FileToUploadWithCollectedInfoDict): @@ -465,6 +466,7 @@ class RequestAttributesDict(TypedDict): bytes: int md5: str adler32: str + checksum: dict[str, Any] is_intermediate_hop: bool diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py index ccf0c11191..b9cd729083 100644 --- a/lib/rucio/core/did.py +++ b/lib/rucio/core/did.py @@ -394,6 +394,7 @@ def __add_files_to_archive( models.DataIdentifier.availability, models.DataIdentifier.adler32, models.DataIdentifier.md5, + models.DataIdentifier.checksum, models.DataIdentifier.is_archive, models.DataIdentifier.constituent, models.DataIdentifier.did_type, @@ -446,6 +447,7 @@ def __add_files_to_archive( 'bytes': file['bytes'], 'adler32': file.get('adler32'), 'md5': file.get('md5'), + 'checksum': file.get('checksum'), 'guid': file.get('guid'), 'length': file.get('events') } @@ -464,6 +466,7 @@ def __add_files_to_archive( 'bytes': row.bytes, 'adler32': row.adler32, 'md5': row.md5, + 'checksum': row.checksum, 'guid': row.guid, 'length': row.events } @@ -563,6 +566,7 @@ def __add_files_to_dataset( if rse_id: # Tier-0 uses this old work-around to register replicas on the RSE # in the same call as attaching them to a dataset + # Note: add_replicas will modify files by filling in checksum dict from legacy checksums, if not present. rucio.core.replica.add_replicas(rse_id=rse_id, files=files.values(), dataset_meta=dataset_meta, account=account, session=session) @@ -576,6 +580,7 @@ def __add_files_to_dataset( models.DataIdentifier.availability, models.DataIdentifier.adler32, models.DataIdentifier.md5, + models.DataIdentifier.checksum, models.DataIdentifier.is_archive, models.DataIdentifier.did_type, ).outerjoin_from( @@ -599,7 +604,6 @@ def __add_files_to_dataset( files_to_add = {} for row in session.execute(stmt): file = files[row.scope, row.name] - if row.did_scope is None: raise exception.DataIdentifierNotFound(f"Data identifier '{row.scope}:{row.name}' not found") @@ -611,7 +615,7 @@ def __add_files_to_dataset( # Check meta-data, if provided row_dict = row._asdict() - for key in ['bytes', 'adler32', 'md5']: + for key in ['bytes', 'adler32', 'md5', 'checksum']: if key in file and str(file[key]) != str(row_dict[key]): raise exception.FileConsistencyMismatch(key + " mismatch for '%(scope)s:%(name)s': " % row_dict + str(file.get(key)) + '!=' + str(row_dict[key])) @@ -633,6 +637,7 @@ def __add_files_to_dataset( 'bytes': row.bytes, 'adler32': row.adler32, 'md5': row.md5, + 'checksum': row.checksum, 'guid': row.guid, 'events': row.events, 'did_type': DIDType.DATASET, @@ -1264,6 +1269,7 @@ def detach_dids( bytes=associ_did.bytes, adler32=associ_did.adler32, md5=associ_did.md5, + checksum=associ_did.checksum, guid=associ_did.guid, events=associ_did.events, rule_evaluation=associ_did.rule_evaluation, @@ -1435,7 +1441,7 @@ def list_content( for tmp_did in session.execute(stmt).yield_per(5).scalars(): children_found = True yield {'scope': tmp_did.child_scope, 'name': tmp_did.child_name, 'type': tmp_did.child_type, - 'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5} + 'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5, 'checksum': tmp_did.checksum} if not children_found: # Raise exception if the DID doesn't exist __get_did(scope=scope, name=name, session=session) @@ -1465,7 +1471,7 @@ def list_content_history( for tmp_did in session.execute(stmt).yield_per(5).scalars(): yield {'scope': tmp_did.child_scope, 'name': tmp_did.child_name, 'type': tmp_did.child_type, - 'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5, + 'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5, 'checksum': tmp_did.checksum, 'deleted_at': tmp_did.deleted_at, 'created_at': tmp_did.created_at, 'updated_at': tmp_did.updated_at} except NoResultFound: @@ -1722,7 +1728,8 @@ def list_files(scope: "InternalScope", name: str, long: bool = False, *, session models.DataIdentifier.guid, models.DataIdentifier.events, models.DataIdentifier.lumiblocknr, - models.DataIdentifier.did_type + models.DataIdentifier.did_type, + models.DataIdentifier.checksum ).with_hint( models.DataIdentifier, 'INDEX(DIDS DIDS_PK)', @@ -1732,16 +1739,15 @@ def list_files(scope: "InternalScope", name: str, long: bool = False, *, session models.DataIdentifier.name == name) ) did = session.execute(stmt).one() - if did[7] == DIDType.FILE: if long: yield {'scope': did[0], 'name': did[1], 'bytes': did[2], 'adler32': did[3], 'guid': did[4] and did[4].upper(), - 'events': did[5], 'lumiblocknr': did[6]} + 'events': did[5], 'lumiblocknr': did[6], 'checksum': did[8]} else: yield {'scope': did[0], 'name': did[1], 'bytes': did[2], 'adler32': did[3], 'guid': did[4] and did[4].upper(), - 'events': did[5]} + 'events': did[5], 'checksum': did[8]} else: cnt_query = select( models.DataIdentifierAssociation.child_scope, @@ -1760,6 +1766,7 @@ def list_files(scope: "InternalScope", name: str, long: bool = False, *, session models.DataIdentifierAssociation.child_type, models.DataIdentifierAssociation.bytes, models.DataIdentifierAssociation.adler32, + models.DataIdentifierAssociation.checksum, models.DataIdentifierAssociation.guid, models.DataIdentifierAssociation.events, models.DataIdentifier.lumiblocknr @@ -1778,6 +1785,7 @@ def list_files(scope: "InternalScope", name: str, long: bool = False, *, session models.DataIdentifierAssociation.child_type, models.DataIdentifierAssociation.bytes, models.DataIdentifierAssociation.adler32, + models.DataIdentifierAssociation.checksum, models.DataIdentifierAssociation.guid, models.DataIdentifierAssociation.events, bindparam("lumiblocknr", None) @@ -1796,16 +1804,16 @@ def list_files(scope: "InternalScope", name: str, long: bool = False, *, session models.DataIdentifierAssociation.name == n) ) - for child_scope, child_name, child_type, bytes_, adler32, guid, events, lumiblocknr in session.execute(stmt).yield_per(500): + for child_scope, child_name, child_type, bytes_, adler32, checksum, guid, events, lumiblocknr in session.execute(stmt).yield_per(500): if long: yield {'scope': child_scope, 'name': child_name, - 'bytes': bytes_, 'adler32': adler32, + 'bytes': bytes_, 'adler32': adler32, 'checksum': checksum, 'guid': guid and guid.upper(), 'events': events, 'lumiblocknr': lumiblocknr} else: yield {'scope': child_scope, 'name': child_name, - 'bytes': bytes_, 'adler32': adler32, + 'bytes': bytes_, 'adler32': adler32, 'checksum': checksum, 'guid': guid and guid.upper(), 'events': events} else: @@ -1953,7 +1961,7 @@ def get_did(scope: "InternalScope", name: str, dynamic_depth: "Optional[DIDType] if did.did_type == DIDType.FILE: return {'scope': did.scope, 'name': did.name, 'type': did.did_type, 'account': did.account, 'bytes': bytes_, 'length': 1, - 'md5': did.md5, 'adler32': did.adler32} + 'md5': did.md5, 'adler32': did.adler32, 'checksum': did.checksum} else: return {'scope': did.scope, 'name': did.name, 'type': did.did_type, 'account': did.account, 'open': did.is_open, @@ -2638,7 +2646,7 @@ def list_archive_content( for tmp_did in session.execute(stmt).yield_per(5).scalars(): yield {'scope': tmp_did.child_scope, 'name': tmp_did.child_name, - 'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5} + 'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5, 'checksum': tmp_did.checksum} except NoResultFound: raise exception.DataIdentifierNotFound(f"Data identifier '{scope}:{name}' not found") @@ -2905,6 +2913,7 @@ def insert_content_history( models.DataIdentifierAssociation.bytes, models.DataIdentifierAssociation.adler32, models.DataIdentifierAssociation.md5, + models.DataIdentifierAssociation.checksum, models.DataIdentifierAssociation.guid, models.DataIdentifierAssociation.events, models.DataIdentifierAssociation.rule_evaluation, @@ -2926,6 +2935,7 @@ def insert_content_history( bytes=cont.bytes, adler32=cont.adler32, md5=cont.md5, + checksum=cont.checksum, guid=cont.guid, events=cont.events, rule_evaluation=cont.rule_evaluation, @@ -2961,6 +2971,7 @@ def insert_deleted_dids(filter_: "ColumnExpressionArgument[bool]", *, session: " models.DataIdentifier.length, models.DataIdentifier.md5, models.DataIdentifier.adler32, + models.DataIdentifier.checksum, models.DataIdentifier.expired_at, models.DataIdentifier.purge_replicas, models.DataIdentifier.deleted_at, @@ -3007,6 +3018,7 @@ def insert_deleted_dids(filter_: "ColumnExpressionArgument[bool]", *, session: " length=did.length, md5=did.md5, adler32=did.adler32, + checksum=did.checksum, expired_at=did.expired_at, purge_replicas=did.purge_replicas, deleted_at=datetime.utcnow(), diff --git a/lib/rucio/core/did_meta_plugins/did_column_meta.py b/lib/rucio/core/did_meta_plugins/did_column_meta.py index 331f9f28fb..c9f15ca20d 100644 --- a/lib/rucio/core/did_meta_plugins/did_column_meta.py +++ b/lib/rucio/core/did_meta_plugins/did_column_meta.py @@ -27,6 +27,7 @@ from rucio.core.did_meta_plugins.filter_engine import FilterEngine from rucio.db.sqla import models from rucio.db.sqla.constants import DIDType +from rucio.db.sqla.models import normalize_checksums from rucio.db.sqla.session import read_session, stream_session, transactional_session if TYPE_CHECKING: @@ -68,7 +69,9 @@ def get_metadata( try: row = session.query(models.DataIdentifier).filter_by(scope=scope, name=name). \ with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one() - return row.to_dict() + row_dict = row.to_dict() + row_dict['checksum'] = normalize_checksums(md5=row_dict.get('md5'), adler32=row_dict.get('adler32'), checksum=row_dict.get('checksum')) + return row_dict except NoResultFound: raise exception.DataIdentifierNotFound(f"Data identifier '{scope}:{name}' not found") diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py index d59a14c5ca..584c1943ab 100644 --- a/lib/rucio/core/replica.py +++ b/lib/rucio/core/replica.py @@ -51,6 +51,7 @@ from rucio.core.rse_expression_parser import parse_expression from rucio.db.sqla import filter_thread_work, models from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, BadPFNStatus, DIDAvailability, DIDType, ReplicaState, RuleState +from rucio.db.sqla.models import normalize_checksums from rucio.db.sqla.session import BASE, DEFAULT_SCHEMA_NAME, read_session, stream_session, transactional_session from rucio.db.sqla.util import temp_table_mngr from rucio.rse import rsemanager as rsemgr @@ -806,7 +807,7 @@ def _list_files_wo_replicas( files_wo_replica: "Iterable[dict[str, Any]]", *, session: "Session" -) -> 'Iterator[tuple[str, str, int, str, str]]': +) -> 'Iterator[tuple[str, str, int, str, str, str]]': if files_wo_replica: file_wo_clause = [] for file in sorted(files_wo_replica, key=lambda f: (f['scope'], f['name'])): @@ -817,7 +818,8 @@ def _list_files_wo_replicas( models.DataIdentifier.name, models.DataIdentifier.bytes, models.DataIdentifier.md5, - models.DataIdentifier.adler32 + models.DataIdentifier.adler32, + models.DataIdentifier.checksum ).with_hint( models.DataIdentifier, 'INDEX(DIDS DIDS_PK)', @@ -826,8 +828,8 @@ def _list_files_wo_replicas( and_(models.DataIdentifier.did_type == DIDType.FILE, or_(*file_wo_clause)) ) - for scope, name, bytes_, md5, adler32 in session.execute(stmt): - yield scope, name, bytes_, md5, adler32 + for scope, name, bytes_, md5, adler32, checksum in session.execute(stmt): + yield scope, name, bytes_, md5, adler32, checksum def get_vp_endpoint() -> str: @@ -1052,7 +1054,7 @@ def _list_replicas( for _, replica_group in groupby(replicas, key=lambda x: (x[0], x[1])): # Group by scope/name file = {} pfns = {} - for scope, name, archive_scope, archive_name, bytes_, md5, adler32, path, state, rse_id, rse, rse_type, volatile in replica_group: + for scope, name, archive_scope, archive_name, bytes_, md5, adler32, checksum, path, state, rse_id, rse, rse_type, volatile in replica_group: if isinstance(archive_scope, str): archive_scope = InternalScope(archive_scope, from_external=False) @@ -1060,8 +1062,9 @@ def _list_replicas( # it is the first row in the scope/name group if not file: + checksum = normalize_checksums(md5=md5, adler32=adler32, checksum=checksum) file['scope'], file['name'] = scope, name - file['bytes'], file['md5'], file['adler32'] = bytes_, md5, adler32 + file['bytes'], file['md5'], file['adler32'], file['checksum'] = bytes_, checksum.get('md5'), checksum.get('adler32'), checksum file['pfns'], file['rses'], file['states'] = {}, {}, {} if resolve_parents: file['parents'] = ['%s:%s' % (parent['scope'].internal, parent['name']) @@ -1177,13 +1180,15 @@ def _list_replicas( if file: yield file - for scope, name, bytes_, md5, adler32 in _list_files_wo_replicas(files_wo_replica, session=session): + for scope, name, bytes_, md5, adler32, checksum in _list_files_wo_replicas(files_wo_replica, session=session): + checksum = normalize_checksums(md5, adler32, checksum) yield { 'scope': scope, 'name': name, 'bytes': bytes_, - 'md5': md5, - 'adler32': adler32, + 'md5': checksum.get(md5), + 'adler32': checksum.get('adler32'), + 'checksum': checksum, 'pfns': {}, 'rses': defaultdict(list) } @@ -1246,6 +1251,7 @@ def _replicas_filter_subquery(): models.RSEFileAssociation.bytes, models.RSEFileAssociation.md5, models.RSEFileAssociation.adler32, + models.RSEFileAssociation.checksum, models.RSE.id.label('rse_id'), models.RSE.rse.label('rse_name'), models.RSE.rse_type, @@ -1329,6 +1335,7 @@ def _list_replicas_for_collection_files_stmt( replicas_subquery.c.bytes, replicas_subquery.c.md5, replicas_subquery.c.adler32, + replicas_subquery.c.checksum, replicas_subquery.c.path, replicas_subquery.c.state, replicas_subquery.c.rse_id, @@ -1359,6 +1366,7 @@ def _list_replicas_for_constituents_stmt( models.ConstituentAssociation.bytes, models.ConstituentAssociation.md5, models.ConstituentAssociation.adler32, + models.ConstituentAssociation.checksum, replicas_subquery.c.path, replicas_subquery.c.state, replicas_subquery.c.rse_id, @@ -1401,6 +1409,7 @@ def _list_replicas_for_input_files_stmt( models.DataIdentifier.bytes, models.DataIdentifier.md5, models.DataIdentifier.adler32, + models.DataIdentifier.checksum, replicas_subquery.c.path, replicas_subquery.c.state, replicas_subquery.c.rse_id, @@ -1594,6 +1603,7 @@ def __bulk_add_new_file_dids( account=file.get('account') or account, did_type=DIDType.FILE, bytes=file['bytes'], md5=file.get('md5'), adler32=file.get('adler32'), + checksum=file.get('checksum'), is_new=None) new_did.save(session=session, flush=False) @@ -1654,6 +1664,7 @@ def __bulk_add_file_dids( models.DataIdentifier.bytes, models.DataIdentifier.md5, models.DataIdentifier.adler32, + models.DataIdentifier.checksum, ).with_hint( models.DataIdentifier, 'INDEX(DIDS DIDS_PK)', @@ -1751,7 +1762,8 @@ def __bulk_add_replicas( 'state': ReplicaState(file.get('state', 'A')), 'md5': file.get('md5'), 'adler32': file.get('adler32'), 'lock_cnt': file.get('lock_cnt', 0), - 'tombstone': file.get('tombstone') or default_tombstone}) + 'tombstone': file.get('tombstone') or default_tombstone, + 'checksum': file.get('checksum')}) try: stmt = insert( models.RSEFileAssociation @@ -1808,6 +1820,11 @@ def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', if not replica_rse['deterministic']: raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %s ' % (replica_rse['rse'])) + # ensure files contain checksum dictionary + for item in files: + checksum = normalize_checksums(item.get('md5'), item.get('adler32'), item.get('checksum')) + if checksum: + item['checksum'] = checksum __bulk_add_file_dids(files=files, account=account, dataset_meta=dataset_meta, session=session) @@ -1868,6 +1885,7 @@ def add_replica( meta: Optional[dict[str, Any]] = None, rules: Optional[list[dict[str, Any]]] = None, tombstone: "Optional[datetime]" = None, + checksum: Optional[dict[str, Any]] = None, *, session: "Session" ) -> list[dict[str, Any]]: @@ -1885,6 +1903,7 @@ def add_replica( :param meta: Meta-data associated with the file. Represented as key/value pairs in a dictionary. :param rules: Replication rules associated with the file. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]. :param tombstone: If True, create replica with a tombstone. + :param checksum: checksum dictionary i.e. {'md5': xxxx, 'adler32': yyyyy} :param session: The database session in use. :returns: list of replicas. @@ -1892,7 +1911,7 @@ def add_replica( meta = meta or {} rules = rules or [] - file = {'scope': scope, 'name': name, 'bytes': bytes_, 'adler32': adler32, 'md5': md5, 'meta': meta, 'rules': rules, 'tombstone': tombstone} + file = {'scope': scope, 'name': name, 'bytes': bytes_, 'adler32': adler32, 'md5': md5, 'meta': meta, 'rules': rules, 'tombstone': tombstone, 'checksum': checksum} if pfn: file['pfn'] = pfn return add_replicas(rse_id=rse_id, files=[file, ], account=account, session=session) @@ -2351,6 +2370,7 @@ def __cleanup_after_replica_deletion( bytes=constituent.bytes, adler32=constituent.adler32, md5=constituent.md5, + checksum=constituent.checksum, guid=constituent.guid, length=constituent.length, updated_at=constituent.updated_at, @@ -3015,6 +3035,7 @@ def get_and_lock_file_replicas_for_dataset( models.DataIdentifierAssociation.bytes, models.DataIdentifierAssociation.md5, models.DataIdentifierAssociation.adler32, + models.DataIdentifierAssociation.checksum, ).where( and_(models.DataIdentifierAssociation.scope == scope, models.DataIdentifierAssociation.name == name) @@ -3040,12 +3061,13 @@ def get_and_lock_file_replicas_for_dataset( thread_id=thread_id, hash_variable='child_name') - for child_scope, child_name, bytes_, md5, adler32 in session.execute(base_stmt).yield_per(1000): + for child_scope, child_name, bytes_, md5, adler32, checksum in session.execute(base_stmt).yield_per(1000): files[(child_scope, child_name)] = {'scope': child_scope, 'name': child_name, 'bytes': bytes_, 'md5': md5, - 'adler32': adler32} + 'adler32': adler32, + 'checksum': checksum} replicas[(child_scope, child_name)] = [] stmt = stmt.where(or_(*rse_clause)) @@ -3076,13 +3098,14 @@ def get_and_lock_file_replicas_for_dataset( of=models.RSEFileAssociation.lock_cnt ) - for child_scope, child_name, bytes_, md5, adler32, replica in session.execute(stmt).yield_per(1000): + for child_scope, child_name, bytes_, md5, adler32, checksum, replica in session.execute(stmt).yield_per(1000): if (child_scope, child_name) not in files: files[(child_scope, child_name)] = {'scope': child_scope, 'name': child_name, 'bytes': bytes_, 'md5': md5, - 'adler32': adler32} + 'adler32': adler32, + 'checksum': checksum} if (child_scope, child_name) in replicas: if replica is not None: diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py index f656b13591..f2b14f9358 100644 --- a/lib/rucio/core/request.py +++ b/lib/rucio/core/request.py @@ -117,6 +117,7 @@ def __init__( name: str, md5: str, adler32: str, + checksum: dict[str, Any], byte_count: int, activity: str, attributes: Optional[Union[str, dict[str, Any]]], @@ -135,6 +136,7 @@ def __init__( self.name = name self.md5 = md5 self.adler32 = adler32 + self.checksum = checksum self.byte_count = byte_count self.activity = activity self._dict_attributes = None @@ -383,6 +385,7 @@ def temp_serializer(obj): 'bytes': request['attributes']['bytes'], 'md5': request['attributes']['md5'], 'adler32': request['attributes']['adler32'], + 'checksum': request['attributes'].get('checksum'), 'account': request.get('account', None), 'priority': request['attributes'].get('priority', None), 'requested_at': request.get('requested_at', None), @@ -428,6 +431,7 @@ def temp_serializer(obj): 'bytes': request['attributes']['bytes'], 'checksum-md5': request['attributes']['md5'], 'checksum-adler': request['attributes']['adler32'], + 'checksum-dict': request['attributes'].get('checksum'), 'queued_at': str(datetime.datetime.utcnow())} messages.append({'event_type': transfer_status, @@ -506,6 +510,7 @@ def list_and_mark_transfer_requests_and_source_replicas( models.Request.name, models.Request.md5, models.Request.adler32, + models.Request.checksum, models.Request.bytes, models.Request.activity, models.Request.attributes, @@ -597,6 +602,7 @@ def list_and_mark_transfer_requests_and_source_replicas( sub_requests.c.name, sub_requests.c.md5, sub_requests.c.adler32, + sub_requests.c.checksum, sub_requests.c.bytes, sub_requests.c.activity, sub_requests.c.attributes, @@ -662,13 +668,13 @@ def list_and_mark_transfer_requests_and_source_replicas( ) requests_by_id = {} - for (request_id, req_type, rule_id, scope, name, md5, adler32, byte_count, activity, attributes, previous_attempt_id, source_rse_id, dest_rse_id, account, retry_count, + for (request_id, req_type, rule_id, scope, name, md5, adler32, checksum, byte_count, activity, attributes, previous_attempt_id, source_rse_id, dest_rse_id, account, retry_count, priority, transfertool, requested_at, replica_rse_id, replica_rse_name, file_path, source_ranking, source_url, distance) in session.execute(stmt): request = requests_by_id.get(request_id) if not request: request = RequestWithSources(id_=request_id, request_type=req_type, rule_id=rule_id, scope=scope, name=name, - md5=md5, adler32=adler32, byte_count=byte_count, activity=activity, attributes=attributes, + md5=md5, adler32=adler32, checksum=checksum, byte_count=byte_count, activity=activity, attributes=attributes, previous_attempt_id=previous_attempt_id, dest_rse=rse_collection[dest_rse_id], account=account, retry_count=retry_count, priority=priority, transfertool=transfertool, requested_at=requested_at) @@ -1311,6 +1317,7 @@ def archive_request( bytes=req['bytes'], md5=req['md5'], adler32=req['adler32'], + checksum=req['checksum'], dest_url=req['dest_url'], requested_at=req['requested_at'], submitted_at=req['submitted_at'], @@ -2866,6 +2873,7 @@ def add_monitor_message( 'duration': -1, 'checksum-adler': request.get('adler32', None), 'checksum-md5': request.get('md5', None), + 'checksum-dict': request.get('checksum', None), 'file-size': request.get('bytes', None), 'bytes': request.get('bytes', None), 'guid': None, diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py index e71b0d5d97..99f134c1b4 100644 --- a/lib/rucio/core/rule.py +++ b/lib/rucio/core/rule.py @@ -736,6 +736,7 @@ def add_rules( # 5. Apply the replication rule to create locks, replicas and transfers with METRICS.timer('add_rules.create_locks_replicas_transfers'): + print('__create_locks_replicas_transfers - replicas:', replicas) try: __create_locks_replicas_transfers(datasetfiles=datasetfiles, locks=locks, @@ -1711,13 +1712,14 @@ def update_rule( query = select( models.RSEFileAssociation.md5, models.RSEFileAssociation.bytes, - models.RSEFileAssociation.adler32 + models.RSEFileAssociation.adler32, + models.RSEFileAssociation.checksum ).where( models.RSEFileAssociation.scope == lock.scope, models.RSEFileAssociation.name == lock.name, models.RSEFileAssociation.rse_id == lock.rse_id ) - md5, bytes_, adler32 = session.execute(query).one() + md5, bytes_, adler32, checksum = session.execute(query).one() session.flush() requests = create_transfer_dict( @@ -1730,6 +1732,7 @@ def update_rule( bytes_=bytes_, md5=md5, adler32=adler32, + checksum=checksum, ds_scope=rule.scope, ds_name=rule.name, copy_pin_lifetime=None, @@ -2661,9 +2664,10 @@ def update_rules_for_bad_replica( bytes_ = replica.bytes md5 = replica.md5 adler32 = replica.adler32 + checksum = replica.checksum request_core.queue_requests(requests=[create_transfer_dict(dest_rse_id=rse_id, request_type=RequestType.TRANSFER, - scope=scope, name=name, rule=rule, lock=lock, bytes_=bytes_, md5=md5, adler32=adler32, + scope=scope, name=name, rule=rule, lock=lock, bytes_=bytes_, md5=md5, adler32=adler32, checksum=checksum, ds_scope=ds_scope, ds_name=ds_name, copy_pin_lifetime=None, activity='Recovery', session=session)], session=session) lock.state = LockState.REPLICATING if rule.state == RuleState.SUSPENDED: @@ -3309,7 +3313,7 @@ def __find_missing_locks_and_create_them( logger(logging.DEBUG, "Finding missing locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt) mod_datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}] - # Files are in the format [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':}] + # Files are in the format [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':, 'checksum':}] for dataset in datasetfiles: mod_files = [] @@ -3888,7 +3892,7 @@ def __resolve_did_to_locks_and_replicas( """ datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}] - # Files are in the format [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':}] + # Files are in the format [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':, 'checksum':}] locks = {} # {(scope,name): [SQLAlchemy]} replicas = {} # {(scope, name): [SQLAlchemy]} source_replicas = {} # {(scope, name): [rse_id] @@ -3900,7 +3904,8 @@ def __resolve_did_to_locks_and_replicas( 'name': did.name, 'bytes': did.bytes, 'md5': did.md5, - 'adler32': did.adler32}]}] + 'adler32': did.adler32, + 'checksum': did.checksum}]}] locks[(did.scope, did.name)] = rucio.core.lock.get_replica_locks(scope=did.scope, name=did.name, nowait=nowait, restrict_rses=restrict_rses, session=session) replicas[(did.scope, did.name)] = rucio.core.replica.get_and_lock_file_replicas(scope=did.scope, name=did.name, nowait=nowait, restrict_rses=restrict_rses, session=session) if source_rses: @@ -3911,7 +3916,7 @@ def __resolve_did_to_locks_and_replicas( locks = rucio.core.lock.get_files_and_replica_locks_of_dataset(scope=did.scope, name=did.name, nowait=nowait, restrict_rses=restrict_rses, only_stuck=True, session=session) for file in locks: file_did = rucio.core.did.get_did(scope=file[0], name=file[1], session=session) - files.append({'scope': file[0], 'name': file[1], 'bytes': file_did['bytes'], 'md5': file_did['md5'], 'adler32': file_did['adler32']}) + files.append({'scope': file[0], 'name': file[1], 'bytes': file_did['bytes'], 'md5': file_did['md5'], 'adler32': file_did['adler32'], 'checksum': file_did['checksum']}) replicas[(file[0], file[1])] = rucio.core.replica.get_and_lock_file_replicas(scope=file[0], name=file[1], nowait=nowait, restrict_rses=restrict_rses, session=session) if source_rses: source_replicas[(file[0], file[1])] = rucio.core.replica.get_source_replicas(scope=file[0], name=file[1], source_rses=source_rses, session=session) @@ -3936,7 +3941,7 @@ def __resolve_did_to_locks_and_replicas( locks = dict(list(locks.items()) + list(tmp_locks.items())) for file in tmp_locks: file_did = rucio.core.did.get_did(scope=file[0], name=file[1], session=session) - files.append({'scope': file[0], 'name': file[1], 'bytes': file_did['bytes'], 'md5': file_did['md5'], 'adler32': file_did['adler32']}) + files.append({'scope': file[0], 'name': file[1], 'bytes': file_did['bytes'], 'md5': file_did['md5'], 'adler32': file_did['adler32'], 'checksum': file_did['checksum']}) replicas[(file[0], file[1])] = rucio.core.replica.get_and_lock_file_replicas(scope=file[0], name=file[1], nowait=nowait, restrict_rses=restrict_rses, session=session) if source_rses: source_replicas[(file[0], file[1])] = rucio.core.replica.get_source_replicas(scope=file[0], name=file[1], source_rses=source_rses, session=session) @@ -3994,7 +3999,7 @@ def __resolve_dids_to_locks_and_replicas( """ datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}] - # Files are in the format [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':}] + # Files are in the format [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':, 'checksum':}] locks = {} # {(scope,name): [SQLAlchemy]} replicas = {} # {(scope, name): [SQLAlchemy]} source_replicas = {} # {(scope, name): [rse_id] @@ -4005,11 +4010,13 @@ def __resolve_dids_to_locks_and_replicas( # Prepare the datasetfiles files = [] for did in dids: + print('DID loop: did checksum ', did.checksum) files.append({'scope': did.child_scope, 'name': did.child_name, 'bytes': did.bytes, 'md5': did.md5, - 'adler32': did.adler32}) + 'adler32': did.adler32, + 'checksum': did.checksum}) locks[(did.child_scope, did.child_name)] = [] replicas[(did.child_scope, did.child_name)] = [] source_replicas[(did.child_scope, did.child_name)] = [] @@ -4184,6 +4191,7 @@ def __create_locks_replicas_transfers( preferred_rse_ids=preferred_rse_ids, source_rses=source_rses, session=session) + print('replicas to create:', replicas_to_create) # Add the replicas session.add_all([item for sublist in replicas_to_create.values() for item in sublist]) session.flush() diff --git a/lib/rucio/core/rule_grouping.py b/lib/rucio/core/rule_grouping.py index 28947a7ee6..4cfb69e336 100644 --- a/lib/rucio/core/rule_grouping.py +++ b/lib/rucio/core/rule_grouping.py @@ -185,7 +185,24 @@ def repair_stuck_locks_and_apply_rule_grouping( @transactional_session -def create_transfer_dict(dest_rse_id, request_type, scope, name, rule, lock=None, bytes_=None, md5=None, adler32=None, ds_scope=None, ds_name=None, copy_pin_lifetime=None, activity=None, retry_count=None, *, session: "Session"): +def create_transfer_dict( + dest_rse_id: str, + request_type: str, + scope: str, + name: str, + rule: models.ReplicationRule, + lock=None, bytes_: Optional[int] = None, + md5: Optional[str] = None, + adler32: Optional[str] = None, + checksum: dict[str, Optional[str]] = {}, + ds_scope: Optional[str] = None, + ds_name: Optional[str] = None, + copy_pin_lifetime=None, + activity: Optional[str] = None, + retry_count: Optional[int] = None, + *, + session: "Session" + ) -> dict[str, Any]: """ This method creates a transfer dictionary and returns it @@ -198,6 +215,7 @@ def create_transfer_dict(dest_rse_id, request_type, scope, name, rule, lock=None :param bytes_: The filesize of the file in bytes. :param md5: The md5 checksum of the file. :param adler32: The adler32 checksum of the file. + :param checksum The checksum as a dictionary {type: value} :param ds_scope: Dataset the file belongs to. :param ds_name: Dataset the file belongs to. :param copy_pin_lifetime: Lifetime in the case of STAGIN requests. @@ -213,6 +231,7 @@ def create_transfer_dict(dest_rse_id, request_type, scope, name, rule, lock=None 'bytes': bytes_, 'md5': md5, 'adler32': adler32, + 'checksum': checksum, 'priority': rule.priority, # 'allow_tape_source': has_account_attribute(account=rule.account, key='admin', session=session)} 'allow_tape_source': True} @@ -987,6 +1006,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab bytes_=file['bytes'], md5=file['md5'], adler32=file['adler32'], + checksum=file['checksum'], ds_scope=dataset['scope'], ds_name=dataset['name'], copy_pin_lifetime=copy_pin_lifetime, @@ -1032,6 +1052,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab bytes_=file['bytes'], md5=file['md5'], adler32=file['adler32'], + checksum=file['checksum'], ds_scope=dataset['scope'], ds_name=dataset['name'], copy_pin_lifetime=copy_pin_lifetime, @@ -1066,6 +1087,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab bytes_=file['bytes'], md5=file['md5'], adler32=file['adler32'], + checksum=file['checksum'], ds_scope=dataset['scope'], ds_name=dataset['name'], session=session)) @@ -1099,6 +1121,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab bytes_=file['bytes'], md5=file['md5'], adler32=file['adler32'], + checksum=file['checksum'], state=ReplicaState.COPYING if (available_source_replica and availability_write) else ReplicaState.UNAVAILABLE) if rse_id not in replicas_to_create: replicas_to_create[rse_id] = [] @@ -1127,6 +1150,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab bytes_=file['bytes'], md5=file['md5'], adler32=file['adler32'], + checksum=file['checksum'], ds_scope=dataset['scope'], ds_name=dataset['name'], session=session)) @@ -1141,6 +1165,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab bytes_=file['bytes'], md5=file['md5'], adler32=file['adler32'], + checksum=file['checksum'], ds_scope=dataset['scope'], ds_name=dataset['name'], session=session)) @@ -1188,17 +1213,18 @@ def __create_lock(rule, rse_id, scope, name, bytes_, state, existing_replica, lo return new_lock -def __create_replica(rse_id, scope, name, bytes_, state, md5, adler32, logger=logging.log): +def __create_replica(rse_id: str, scope: str, name: str, bytes_: int, state: ReplicaState, md5: Optional[str], adler32: Optional[str], checksum: dict[str, Optional[str]], logger=logging.log): """ Create and return a new SQLAlchemy replica object. :param rse_id: RSE id of the replica. :param scope: Scope of the replica. :param name: Name of the replica. - :param bytes_: Bytes of the replica. + :param bytes_: Bytes of the replica. :param state: State of the replica. :param md5: MD5 checksum of the replica. :param adler32: ADLER32 checksum of the replica. + :param checksum: checksum dictionary {type: value} :param logger: Optional decorated logger that can be passed from the calling daemons or servers. """ @@ -1208,6 +1234,7 @@ def __create_replica(rse_id, scope, name, bytes_, state, md5, adler32, logger=lo bytes=bytes_, md5=md5, adler32=adler32, + checksum=checksum, tombstone=None, state=state, lock_cnt=0) @@ -1252,6 +1279,7 @@ def __update_lock_replica_and_create_transfer(lock, replica, rule, dataset, tran bytes_=replica.bytes, md5=replica.md5, adler32=replica.adler32, + checksum=replica.checksum, ds_scope=dataset['scope'], ds_name=dataset['name'], copy_pin_lifetime=copy_pin_lifetime, @@ -1266,6 +1294,7 @@ def __update_lock_replica_and_create_transfer(lock, replica, rule, dataset, tran bytes_=replica.bytes, md5=replica.md5, adler32=replica.adler32, + checksum=replica.checksum, ds_scope=dataset['scope'], ds_name=dataset['name'], request_type=RequestType.TRANSFER, @@ -1363,7 +1392,7 @@ def apply_rule(did, rule, rses, source_rses, rseselector, *, session: "Session", # to align code with cases below, create file dict file = {'name': did.name, 'scope': did.scope, - 'bytes': did.bytes, 'md5': did.md5, 'adler32': did.adler32} + 'bytes': did.bytes, 'md5': did.md5, 'adler32': did.adler32, 'checksum': did.checksum} # calculate target RSEs rse_coverage = {replica.rse_id: file['bytes'] for replica in replicas[(file['scope'], file['name'])]} @@ -1486,7 +1515,7 @@ def apply_rule(did, rule, rses, source_rses, rseselector, *, session: "Session", for p in range(npartitions): # prnt(('processing partition ', p, npartitions)) - # files is [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':}] + # files is [{'scope':, 'name':, 'bytes':, 'md5':, 'adler32':, 'checksum': }] # locks is {(scope,name): [SQLAlchemy]} # replicas = {(scope, name): [SQLAlchemy]} # source replicas is {(scope, name): [SQLAlchemy]} diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py index f82d9428bf..0e2db1e858 100644 --- a/lib/rucio/core/transfer.py +++ b/lib/rucio/core/transfer.py @@ -669,6 +669,7 @@ def _create_transfer_definitions( name=rws.name, md5=rws.md5, adler32=rws.adler32, + checksum=rws.checksum, byte_count=rws.byte_count, activity=rws.activity, attributes={ @@ -680,6 +681,7 @@ def _create_transfer_definitions( 'bytes': rws.byte_count, 'md5': rws.md5, 'adler32': rws.adler32, + 'checksum': rws.checksum, 'priority': None, 'allow_tape_source': True }, diff --git a/lib/rucio/db/sqla/migrate_repo/versions/8ab4d628cffb_add_generic_checksum_column.py b/lib/rucio/db/sqla/migrate_repo/versions/8ab4d628cffb_add_generic_checksum_column.py new file mode 100644 index 0000000000..51020580c6 --- /dev/null +++ b/lib/rucio/db/sqla/migrate_repo/versions/8ab4d628cffb_add_generic_checksum_column.py @@ -0,0 +1,60 @@ +# Copyright European Organization for Nuclear Research (CERN) since 2012 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""add generic checksum column""" # noqa: D400, D415 + +import sqlalchemy as sa +from alembic import context +from alembic.op import add_column, drop_column + +from rucio.db.sqla.types import JSON + +# Alembic revision identifiers +revision = '8ab4d628cffb' +down_revision = '3b943000da18' + + +def upgrade(): + """Upgrade the database to this revision.""" + + if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']: + schema = context.get_context().version_table_schema if context.get_context().version_table_schema else '' + add_column('replicas', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('dids', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('deleted_dids', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('quarantined_replicas', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('quarantined_replicas_history', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('contents', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('archive_contents', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('archive_contents_history', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('contents_history', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('requests', sa.Column('checksum', JSON(), nullable=True), schema=schema) + add_column('requests_history', sa.Column('checksum', JSON(), nullable=True), schema=schema) + + +def downgrade(): + """Downgrade the database to the previous revision.""" + if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']: + schema = context.get_context().version_table_schema if context.get_context().version_table_schema else '' + drop_column('replicas', 'checksum', schema=schema) + drop_column('dids', 'checksum', schema=schema) + drop_column('deleted_dids', 'checksum', schema=schema) + drop_column('quarantined_replicas', 'checksum', schema=schema) + drop_column('quarantined_replicas_history', 'checksum', schema=schema) + drop_column('contents', 'checksum', schema=schema) + drop_column('archive_contents', 'checksum', schema=schema) + drop_column('archive_contents_history', 'checksum', schema=schema) + drop_column('contents_history', 'checksum', schema=schema) + drop_column('requests', 'checksum', schema=schema) + drop_column('requests_history', 'checksum', schema=schema) diff --git a/lib/rucio/db/sqla/models.py b/lib/rucio/db/sqla/models.py index d2bd4cd4b7..814b988c20 100644 --- a/lib/rucio/db/sqla/models.py +++ b/lib/rucio/db/sqla/models.py @@ -434,6 +434,7 @@ class DataIdentifier(BASE, ModelBase): purge_replicas: Mapped[bool] = mapped_column(Boolean(name='DIDS_PURGE_RPLCS_CHK', create_constraint=True), server_default='1') deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) # hardcoded meta-data to populate the db events: Mapped[Optional[int]] = mapped_column(BigInteger) guid: Mapped[Optional[uuid.UUID]] = mapped_column(GUID()) @@ -649,6 +650,7 @@ class DeletedDataIdentifier(BASE, ModelBase): is_archive: Mapped[Optional[bool]] = mapped_column(Boolean(name='DEL_DIDS_ARCH_CHK', create_constraint=True)) constituent: Mapped[Optional[bool]] = mapped_column(Boolean(name='DEL_DIDS_CONST_CHK', create_constraint=True)) access_cnt: Mapped[Optional[int]] = mapped_column(Integer()) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) _table_args = (PrimaryKeyConstraint('scope', 'name', name='DELETED_DIDS_PK'), ) @@ -716,6 +718,7 @@ class QuarantinedReplica(BASE, ModelBase): adler32: Mapped[Optional[str]] = mapped_column(String(8)) scope: Mapped[Optional[InternalScope]] = mapped_column(InternalScopeString(common_schema.get_schema_value('SCOPE_LENGTH'))) name: Mapped[Optional[str]] = mapped_column(String(common_schema.get_schema_value('NAME_LENGTH'))) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) _table_args = (PrimaryKeyConstraint('rse_id', 'path', name='QURD_REPLICAS_STATE_PK'), ForeignKeyConstraint(['rse_id'], ['rses.id'], name='QURD_REPLICAS_RSE_ID_FK'), Index('QUARANTINED_REPLICAS_PATH_IDX', 'path', 'rse_id', unique=True)) @@ -732,6 +735,7 @@ class QuarantinedReplicaHistory(BASE, ModelBase): scope: Mapped[Optional[InternalScope]] = mapped_column(InternalScopeString(common_schema.get_schema_value('SCOPE_LENGTH'))) name: Mapped[Optional[str]] = mapped_column(String(common_schema.get_schema_value('NAME_LENGTH'))) deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) __mapper_args__ = { 'primary_key': [rse_id, path] # Fake primary key for SQLA } @@ -782,6 +786,7 @@ class DataIdentifierAssociation(BASE, ModelBase): guid: Mapped[Optional[uuid.UUID]] = mapped_column(GUID()) events: Mapped[Optional[int]] = mapped_column(BigInteger) rule_evaluation: Mapped[Optional[bool]] = mapped_column(Boolean(name='CONTENTS_RULE_EVALUATION_CHK', create_constraint=True)) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) _table_args = (PrimaryKeyConstraint('scope', 'name', 'child_scope', 'child_name', name='CONTENTS_PK'), ForeignKeyConstraint(['scope', 'name'], ['dids.scope', 'dids.name'], name='CONTENTS_ID_FK'), ForeignKeyConstraint(['child_scope', 'child_name'], ['dids.scope', 'dids.name'], ondelete="CASCADE", name='CONTENTS_CHILD_ID_FK'), @@ -803,6 +808,7 @@ class ConstituentAssociation(BASE, ModelBase): md5: Mapped[Optional[str]] = mapped_column(String(32)) guid: Mapped[Optional[uuid.UUID]] = mapped_column(GUID()) length: Mapped[Optional[int]] = mapped_column(BigInteger) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) _table_args = (PrimaryKeyConstraint('child_scope', 'child_name', 'scope', 'name', name='ARCH_CONTENTS_PK'), ForeignKeyConstraint(['scope', 'name'], ['dids.scope', 'dids.name'], @@ -826,6 +832,7 @@ class ConstituentAssociationHistory(BASE, ModelBase): md5: Mapped[Optional[str]] = mapped_column(String(32)) guid: Mapped[Optional[uuid.UUID]] = mapped_column(GUID()) length: Mapped[Optional[int]] = mapped_column(BigInteger) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) __mapper_args__ = { 'primary_key': [scope, name, child_scope, child_name] # Fake primary key for SQLA } @@ -853,6 +860,7 @@ class DataIdentifierAssociationHistory(BASE, ModelBase): rule_evaluation: Mapped[Optional[bool]] = mapped_column(Boolean(name='CONTENTS_HIST_RULE_EVAL_CHK', create_constraint=True)) did_created_at: Mapped[Optional[datetime]] = mapped_column(DateTime) deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) __mapper_args__ = { 'primary_key': [scope, name, child_scope, child_name] # Fake primary key for SQLA } @@ -1101,6 +1109,7 @@ class RSEFileAssociation(BASE, ModelBase): lock_cnt: Mapped[int] = mapped_column(Integer, server_default='0') accessed_at: Mapped[Optional[datetime]] = mapped_column(DateTime) tombstone: Mapped[Optional[datetime]] = mapped_column(DateTime) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) _table_args = (PrimaryKeyConstraint('scope', 'name', 'rse_id', name='REPLICAS_PK'), ForeignKeyConstraint(['scope', 'name'], ['dids.scope', 'dids.name'], name='REPLICAS_LFN_FK'), ForeignKeyConstraint(['rse_id'], ['rses.id'], name='REPLICAS_RSE_ID_FK'), @@ -1445,6 +1454,7 @@ class Request(BASE, ModelBase): last_processed_at: Mapped[Optional[datetime]] = mapped_column(DateTime) priority: Mapped[Optional[int]] = mapped_column(Integer) transfertool: Mapped[Optional[str]] = mapped_column(String(64)) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) _table_args = (PrimaryKeyConstraint('id', name='REQUESTS_PK'), ForeignKeyConstraint(['scope', 'name'], ['dids.scope', 'dids.name'], name='REQUESTS_DID_FK'), ForeignKeyConstraint(['dest_rse_id'], ['rses.id'], name='REQUESTS_RSES_FK'), @@ -1518,6 +1528,7 @@ class RequestHistory(BASE, ModelBase): requested_at: Mapped[Optional[datetime]] = mapped_column(DateTime) priority: Mapped[Optional[int]] = mapped_column(Integer) transfertool: Mapped[Optional[str]] = mapped_column(String(64)) + checksum: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON()) __mapper_args__ = { 'primary_key': [id] # Fake primary key for SQLA } @@ -1856,3 +1867,32 @@ def unregister_models(engine: Engine) -> None: :returns: None """ BASE.metadata.drop_all(engine) + + +def normalize_checksums(md5: Optional[str] = None, + adler32: Optional[str] = None, + checksum: Optional[Any] = None) -> dict[str, Optional[str]]: + """ + Return a canonical checksum dict or {}. + + Rules: + - If checksum is a non-empty dict (DB canonical JSON), return it as-is (preserve keys). + - If checksum is None or an empty dict, build a checksum dict from available legacy columns (md5/adler32). + - If neither checksum nor legacy columns provide values, return {}. + + The returned dict may contain other checksum types in the future (e.g. 'sha1'); those are preserved. + """ + # If DB column returns a non-empty dict, prefer it (canonical) + if isinstance(checksum, dict) and checksum: + return checksum + + # If checksum is truthy but not a dict, don't attempt parsing here. + # We treat non-dict values as absent and fall back to legacy columns. + # Synthesize from legacy columns when DB checksum is absent or empty + out: dict[str, Optional[str]] = {} + if md5: + out['md5'] = md5 + if adler32: + out['adler32'] = adler32 + + return out diff --git a/lib/rucio/db/sqla/types.py b/lib/rucio/db/sqla/types.py index 20db8ecaa6..1d800ee96d 100644 --- a/lib/rucio/db/sqla/types.py +++ b/lib/rucio/db/sqla/types.py @@ -18,6 +18,7 @@ from sqlalchemy.dialects.mysql import BINARY from sqlalchemy.dialects.oracle import CLOB, RAW from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.dialects.sqlite import JSON as JSONSQ from sqlalchemy.sql import operators from sqlalchemy.types import CHAR, String, TypeDecorator @@ -136,6 +137,8 @@ def load_dialect_impl(self, dialect): return dialect.type_descriptor(types.JSON()) elif dialect.name == 'oracle': return dialect.type_descriptor(CLOB()) + elif dialect.name == 'sqlite': + return dialect.type_descriptor(JSONSQ()) else: return dialect.type_descriptor(String()) diff --git a/pyproject.client.toml b/pyproject.client.toml index d84f0d2d6a..76e5db72df 100644 --- a/pyproject.client.toml +++ b/pyproject.client.toml @@ -33,7 +33,7 @@ dependencies = [ 'rich', 'typing_extensions' ] -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [ {name = "Rucio", email = "rucio-contact@cern.ch"}, ] @@ -52,8 +52,8 @@ classifiers = [ 'Natural Language :: English', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ] [project.optional-dependencies] diff --git a/pyproject.server.toml b/pyproject.server.toml index f0397d9475..8fc505a0a9 100644 --- a/pyproject.server.toml +++ b/pyproject.server.toml @@ -34,7 +34,7 @@ dependencies = [ 'python-magic<=0.4.27', 'paramiko<=4.0.0', 'boto3<=1.40.64', - 'sqlalchemy<=2.0.44', + 'sqlalchemy==2.1.0b2', 'alembic<=1.16.5', 'pymemcache<=4.0.0', 'python-dateutil<=2.9.0.post0', @@ -47,7 +47,7 @@ dependencies = [ 'oic<=1.7.0', 'prometheus_client<=0.23.1', ] -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [ {name = "Rucio", email = "rucio-contact@cern.ch"}, ] @@ -66,8 +66,8 @@ classifiers = [ 'Natural Language :: English', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Environment :: No Input/Output (Daemon)', ] diff --git a/pyproject.toml b/pyproject.toml index afb4f4f944..075f7c92c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ 'python-magic<=0.4.27', 'paramiko<=4.0.0', 'boto3<=1.40.64', - 'sqlalchemy<=2.0.44', + 'sqlalchemy==2.1.0b2', 'alembic<=1.16.5', 'pymemcache<=4.0.0', 'python-dateutil<=2.9.0.post0', @@ -38,7 +38,7 @@ dependencies = [ 'oic<=1.7.0', 'prometheus_client<=0.23.1', ] -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [ {name = "Rucio", email = "rucio-contact@cern.ch"}, ] @@ -56,8 +56,8 @@ classifiers = [ 'Natural Language :: English', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Environment :: No Input/Output (Daemon)', ] @@ -120,7 +120,7 @@ exclude_also = [ [tool.ruff] line-length = 256 -target-version = "py39" +target-version = "py310" extend-include = [ "bin/*", diff --git a/requirements/requirements.dev.txt b/requirements/requirements.dev.txt index a1ec2bf370..23a288b56f 100644 --- a/requirements/requirements.dev.txt +++ b/requirements/requirements.dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --strip-extras requirements.dev.in @@ -174,10 +174,6 @@ globus-sdk==4.1.0 # via -r requirements.server.txt google-auth==2.42.1 # via -r requirements.server.txt -greenlet==3.2.4 - # via - # -r requirements.server.txt - # sqlalchemy gssapi==1.10.1 # via # -r requirements.server.txt @@ -187,11 +183,6 @@ idna==3.11 # -r requirements.server.txt # requests # yarl -importlib-metadata==8.7.0 - # via - # -r requirements.server.txt - # build - # flask iniconfig==2.1.0 # via pytest invoke==2.2.1 @@ -461,7 +452,7 @@ six==1.17.0 # -r requirements.server.txt # pyjwkest # python-dateutil -sqlalchemy==2.0.44 +sqlalchemy==2.1.0b2 # via # -r requirements.server.txt # alembic @@ -475,7 +466,7 @@ stomp-py==8.2.0 # via -r requirements.server.txt tabulate==0.9.0 # via -r requirements.server.txt -tomli==2.3.0 +tomli==2.4.1 # via # -r requirements.server.txt # alembic @@ -551,10 +542,6 @@ yarl==1.22.0 # via # -r requirements.server.txt # aiohttp -zipp==3.23.0 - # via - # -r requirements.server.txt - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/requirements.server.in b/requirements/requirements.server.in index 18bff33af2..b72179e82d 100644 --- a/requirements/requirements.server.in +++ b/requirements/requirements.server.in @@ -4,7 +4,7 @@ urllib3==1.26.19, <2 # HTTP library with dogpile.cache==1.2.2 # Caching API plugins (1.1.2 is the first version to support pymemcache) tabulate==0.9.0 # Pretty-print tabular data jsonschema==4.25.1 # For JSON schema validation (Policy modules) -SQLAlchemy==2.0.44 # DB backend +SQLAlchemy==2.1.0b2 # DB backend alembic==1.16.5 # Lightweight database migration tool for SQLAlchemy; 1.17.1 requires Python 3.10+ pymemcache==4.0.0 # A comprehensive, fast, pure-Python memcached client (Used by Dogpile) python-dateutil==2.9.0.post0 # Extensions to the standard datetime module diff --git a/requirements/requirements.server.txt b/requirements/requirements.server.txt index 5d41b12664..001bd8dc6d 100644 --- a/requirements/requirements.server.txt +++ b/requirements/requirements.server.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --strip-extras requirements.server.in @@ -88,16 +88,12 @@ globus-sdk==4.1.0 # via -r requirements.server.in google-auth==2.42.1 # via -r requirements.server.in -greenlet==3.2.4 - # via sqlalchemy gssapi==1.10.1 # via pyspnego idna==3.11 # via # requests # yarl -importlib-metadata==8.7.0 - # via flask invoke==2.2.1 # via paramiko isodate==0.7.2 @@ -250,7 +246,7 @@ six==1.17.0 # via # pyjwkest # python-dateutil -sqlalchemy==2.0.44 +sqlalchemy==2.1.0b2 # via # -r requirements.server.in # alembic @@ -262,7 +258,7 @@ stomp-py==8.2.0 # via -r requirements.server.in tabulate==0.9.0 # via -r requirements.server.in -tomli==2.3.0 +tomli==2.4.1 # via alembic typing-extensions==4.15.0 # via @@ -303,5 +299,3 @@ xmlsec==1.3.16 # python3-saml yarl==1.22.0 # via aiohttp -zipp==3.23.0 - # via importlib-metadata