From 4b2f28c1c5ad0505a2c2475c1d05d2c4f06e52d0 Mon Sep 17 00:00:00 2001 From: Emma <194498507+emmaicos@users.noreply.github.com> Date: Fri, 28 Feb 2025 12:07:20 +0100 Subject: [PATCH 1/4] validate temporal coverage when deprecating data objects --- .../upload/validation/ScopedValidator.scala | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala index 661fff873..fc4891798 100644 --- a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala +++ b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala @@ -113,7 +113,10 @@ private class ScopedValidator(vocab: CpVocab, val metaVocab: CpmetaVocab) extend if allDuplicates.exists(deprecated.contains) then Success(dto) else if dto.autodeprecateSameFilenameObjects then - withDeprecations(dto, deprecated ++ allDuplicates) + if validChangedTemporalCoverage(dto) then + withDeprecations(dto, deprecated ++ allDuplicates) + else + userFail("New temporal coverage must include temporal coverage of the objects being deprecated.") else userFail( @@ -132,6 +135,34 @@ private class ScopedValidator(vocab: CpVocab, val metaVocab: CpmetaVocab) extend .toIndexedSeq .nonEmpty + private def validChangedTemporalCoverage[Dto <: ObjectUploadDto](dto: Dto)(using Envri, DocConn | DobjConn): Boolean = + + val newIntervalIncludesPrevious = for + dataDto <- dto.asOptInstanceOf[DataObjectDto] + timeSeries <- dataDto.specificInfo match + case Right(timeSeries) => Some(timeSeries) + case _ => None + newTimeInterval <- timeSeries.acquisitionInterval + newStart = newTimeInterval.start + newStop = newTimeInterval.stop + previousIris = getPropValueHolders(metaVocab.hasName, vf.createLiteral(dto.fileName)) + .collect{case subj if isCompleted(subj) && !isDeprecated(subj) => subj} + .toIndexedSeq + aquiredByIris = previousIris.map(getSingleUri(_, metaVocab.wasAcquiredBy).result) + .collect{case Some(iri) => iri} + previousStarts = aquiredByIris.map(getSingleInstant(_, metaVocab.prov.startedAtTime).result) + .collect{case Some(iri) => iri} + previousStops = aquiredByIris.map(getSingleInstant(_, metaVocab.prov.endedAtTime).result) + .collect{case Some(iri) => iri} + yield + (previousStarts.isEmpty || !newStart.isAfter(previousStarts.min)) + && (previousStops.isEmpty || !newStop.isBefore(previousStops.max)) + + newIntervalIncludesPrevious.getOrElse(true) + + end validChangedTemporalCoverage + + def growingIsGrowing( dto: ObjectUploadDto, spec: DataObjectSpec, From 99f90a1b635433e61362a453b0826316c0ae3ce4 Mon Sep 17 00:00:00 2001 From: Jonathan Thiry Date: Mon, 3 Mar 2025 13:33:10 +0100 Subject: [PATCH 2/4] Refactor temporal coverage comparaison --- .../upload/validation/ScopedValidator.scala | 33 +++++++------------ .../upload/validation/UploadValidator.scala | 1 + 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala index fc4891798..496f62a50 100644 --- a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala +++ b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala @@ -113,10 +113,7 @@ private class ScopedValidator(vocab: CpVocab, val metaVocab: CpmetaVocab) extend if allDuplicates.exists(deprecated.contains) then Success(dto) else if dto.autodeprecateSameFilenameObjects then - if validChangedTemporalCoverage(dto) then - withDeprecations(dto, deprecated ++ allDuplicates) - else - userFail("New temporal coverage must include temporal coverage of the objects being deprecated.") + withDeprecations(dto, deprecated ++ allDuplicates) else userFail( @@ -135,32 +132,26 @@ private class ScopedValidator(vocab: CpVocab, val metaVocab: CpmetaVocab) extend .toIndexedSeq .nonEmpty - private def validChangedTemporalCoverage[Dto <: ObjectUploadDto](dto: Dto)(using Envri, DocConn | DobjConn): Boolean = + def compareToPreviousTemporalCoverage(dto: DataObjectDto)(using Envri, DocConn | DobjConn): Try[NotUsed] = val newIntervalIncludesPrevious = for dataDto <- dto.asOptInstanceOf[DataObjectDto] - timeSeries <- dataDto.specificInfo match - case Right(timeSeries) => Some(timeSeries) - case _ => None + timeSeries <- dataDto.specificInfo.fold(_ => None, ts => Some(ts)) newTimeInterval <- timeSeries.acquisitionInterval - newStart = newTimeInterval.start - newStop = newTimeInterval.stop - previousIris = getPropValueHolders(metaVocab.hasName, vf.createLiteral(dto.fileName)) + aquiredByIris = getPropValueHolders(metaVocab.hasName, vf.createLiteral(dto.fileName)) .collect{case subj if isCompleted(subj) && !isDeprecated(subj) => subj} .toIndexedSeq - aquiredByIris = previousIris.map(getSingleUri(_, metaVocab.wasAcquiredBy).result) - .collect{case Some(iri) => iri} - previousStarts = aquiredByIris.map(getSingleInstant(_, metaVocab.prov.startedAtTime).result) - .collect{case Some(iri) => iri} - previousStops = aquiredByIris.map(getSingleInstant(_, metaVocab.prov.endedAtTime).result) - .collect{case Some(iri) => iri} + .flatMap(getSingleUri(_, metaVocab.wasAcquiredBy).result) + previousStarts = aquiredByIris.flatMap(getSingleInstant(_, metaVocab.prov.startedAtTime).result) + previousStops = aquiredByIris.flatMap(getSingleInstant(_, metaVocab.prov.endedAtTime).result) yield - (previousStarts.isEmpty || !newStart.isAfter(previousStarts.min)) - && (previousStops.isEmpty || !newStop.isBefore(previousStops.max)) + (previousStarts.isEmpty || !newTimeInterval.start.isAfter(previousStarts.min)) + && (previousStops.isEmpty || !newTimeInterval.stop.isBefore(previousStops.max)) - newIntervalIncludesPrevious.getOrElse(true) + if newIntervalIncludesPrevious.getOrElse(true) then ok + else userFail("New temporal coverage must include temporal coverage of the objects being deprecated.") - end validChangedTemporalCoverage + end compareToPreviousTemporalCoverage def growingIsGrowing( diff --git a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala index 14dd69ffb..31288211f 100644 --- a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala +++ b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala @@ -66,6 +66,7 @@ class UploadValidator(servers: DataObjectInstanceServers): _ <- scoped.growingIsGrowing(meta, spec, submConf) _ <- validateActors(meta) _ <- validateTemporalCoverage(meta, spec) + _ <- scoped.compareToPreviousTemporalCoverage(meta) _ <- noProductionProvenanceIfL0(meta, spec) _ <- validateFormatsByFileExt(meta, spec) amended0 <- validateSpatialCoverage(meta) From e5c11bd83518e2c4c9cae1599dd2115651592b1f Mon Sep 17 00:00:00 2001 From: Emma <194498507+emmaicos@users.noreply.github.com> Date: Mon, 3 Mar 2025 15:16:12 +0100 Subject: [PATCH 3/4] only check temporal coverage if deprecate --- .../cp/meta/services/upload/validation/UploadValidator.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala index 31288211f..4622be393 100644 --- a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala +++ b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala @@ -66,7 +66,10 @@ class UploadValidator(servers: DataObjectInstanceServers): _ <- scoped.growingIsGrowing(meta, spec, submConf) _ <- validateActors(meta) _ <- validateTemporalCoverage(meta, spec) - _ <- scoped.compareToPreviousTemporalCoverage(meta) + _ <- + if meta.autodeprecateSameFilenameObjects then + scoped.compareToPreviousTemporalCoverage(meta) + else ok _ <- noProductionProvenanceIfL0(meta, spec) _ <- validateFormatsByFileExt(meta, spec) amended0 <- validateSpatialCoverage(meta) From 6253532de9ca8f072762b7969114d2d1428d8c9e Mon Sep 17 00:00:00 2001 From: Emma <194498507+emmaicos@users.noreply.github.com> Date: Tue, 4 Mar 2025 11:04:49 +0100 Subject: [PATCH 4/4] handle specified previous versions to deprecate --- .../services/upload/validation/ScopedValidator.scala | 12 ++++++++---- .../services/upload/validation/UploadValidator.scala | 5 +---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala index 496f62a50..4a489b459 100644 --- a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala +++ b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/ScopedValidator.scala @@ -138,12 +138,16 @@ private class ScopedValidator(vocab: CpVocab, val metaVocab: CpmetaVocab) extend dataDto <- dto.asOptInstanceOf[DataObjectDto] timeSeries <- dataDto.specificInfo.fold(_ => None, ts => Some(ts)) newTimeInterval <- timeSeries.acquisitionInterval - aquiredByIris = getPropValueHolders(metaVocab.hasName, vf.createLiteral(dto.fileName)) - .collect{case subj if isCompleted(subj) && !isDeprecated(subj) => subj} + autoDeprecateIris = getPropValueHolders(metaVocab.hasName, vf.createLiteral(dto.fileName)) + .collect{case subj if dto.autodeprecateSameFilenameObjects && isCompleted(subj) && !isDeprecated(subj) => subj} .toIndexedSeq + selectedDeprecateIris = dto.isNextVersionOf.flattenToSeq.iterator.map(prevHash => + vocab.getStaticObject(prevHash)) + .toIndexedSeq + allAquiredByIris = (autoDeprecateIris ++ selectedDeprecateIris).toSet .flatMap(getSingleUri(_, metaVocab.wasAcquiredBy).result) - previousStarts = aquiredByIris.flatMap(getSingleInstant(_, metaVocab.prov.startedAtTime).result) - previousStops = aquiredByIris.flatMap(getSingleInstant(_, metaVocab.prov.endedAtTime).result) + previousStarts = allAquiredByIris.flatMap(getSingleInstant(_, metaVocab.prov.startedAtTime).result) + previousStops = allAquiredByIris.flatMap(getSingleInstant(_, metaVocab.prov.endedAtTime).result) yield (previousStarts.isEmpty || !newTimeInterval.start.isAfter(previousStarts.min)) && (previousStops.isEmpty || !newTimeInterval.stop.isBefore(previousStops.max)) diff --git a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala index 4622be393..31288211f 100644 --- a/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala +++ b/src/main/scala/se/lu/nateko/cp/meta/services/upload/validation/UploadValidator.scala @@ -66,10 +66,7 @@ class UploadValidator(servers: DataObjectInstanceServers): _ <- scoped.growingIsGrowing(meta, spec, submConf) _ <- validateActors(meta) _ <- validateTemporalCoverage(meta, spec) - _ <- - if meta.autodeprecateSameFilenameObjects then - scoped.compareToPreviousTemporalCoverage(meta) - else ok + _ <- scoped.compareToPreviousTemporalCoverage(meta) _ <- noProductionProvenanceIfL0(meta, spec) _ <- validateFormatsByFileExt(meta, spec) amended0 <- validateSpatialCoverage(meta)