From 1b16a002149ae96021edffe03fc8516f1c2fa89c Mon Sep 17 00:00:00 2001 From: jonathanlindahl Date: Fri, 24 Oct 2025 15:11:42 +0200 Subject: [PATCH 1/5] added ccbync4 license and adjusted license pdf and table of contents --- .../se/lu/nateko/cp/data/api/MetaVocab.scala | 1 + .../services/upload/DownloadService.scala | 27 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala b/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala index 2ae538b9..ad6350db 100644 --- a/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala +++ b/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala @@ -66,4 +66,5 @@ object SitesMetaVocab extends MetaVocab(new URI("https://meta.fieldsites.se/onto object CcMetaVocab extends MetaVocab(new URI("https://creativecommons.org/")){ val cc0 = getRelative("publicdomain/zero/1.0/") + val ccbync4 = getRelative("licenses/by-nc/4.0") } diff --git a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala index a087b2b5..b976ceaa 100644 --- a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala +++ b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala @@ -128,11 +128,21 @@ class DownloadService(coreConf: MetaCoreConfig, val upload: UploadService, val r private def destinyToAuxSourcesFlow(implicit envri: Envri): Flow[FileDestiny, FileEntry, NotUsed] = Flow.apply[FileDestiny] .fold(Vector.empty[FileDestiny])(_ :+ _) - .map{dests => - ZipEntry("!TOC.csv") -> destiniesToTocFileSource(dests) - }.concat(Source.single( - ZipEntry("!LICENCE.pdf") -> licenceSource - )) + .flatMapConcat{dests => + val baseSource = Source.single(ZipEntry("!TOC.csv") -> destiniesToTocFileSource(dests)) + + val shouldIncludeLincensePdf = dests.exists( + _.obj.references.licence.exists( + _.toString().contains("creativecommons.org/licenses/by/4.0") + ) + ) + + if (shouldIncludeLincensePdf) { + baseSource.concat(Source.single(ZipEntry("!LICENCE.pdf") -> licenceSource)) + } else { + baseSource + } + } private def singleObjectSource(obj: StaticObject, downloadLogger: DataObject => Unit): Source[ByteString, NotUsed] = { val file = upload.getFile(obj, true) @@ -150,7 +160,7 @@ class DownloadService(coreConf: MetaCoreConfig, val upload: UploadService, val r } private def destiniesToTocFileSource(dests: immutable.Seq[FileDestiny])(using Envri): Source[ByteString, NotUsed] = { - val lines = "Included,File name,PID,Landing page,Omission reason (if any)\n" +: dests.map{dest => + val lines = "Included,File name,PID,Landing page,License,Omission reason (if any)\n" +: dests.map{dest => val presense = if(dest.omissionReason.isEmpty) "Yes" else "No" val omissionReason = dest.omissionReason.getOrElse("") @@ -163,7 +173,8 @@ class DownloadService(coreConf: MetaCoreConfig, val upload: UploadService, val r val hdlProxy = if(dest.obj.doi.isDefined) prox.doi else prox.basic s"$hdlProxy$pid" } - s"$presense,${dest.fileName},${pidOpt.getOrElse("")},$landingPage,$omissionReason\n" + val license = if (!dest.obj.references.licence.isEmpty) dest.obj.references.licence.get.name else "" + s"$presense,${dest.fileName},${pidOpt.getOrElse("")},$landingPage,$license,$omissionReason\n" } Source(lines.map(ByteString.apply)) } @@ -227,7 +238,7 @@ private object ZeroDestiny extends Destiny{ object DownloadService: - val publicDomainLicences = Set(CcMetaVocab.cc0) + val publicDomainLicences = Set(CcMetaVocab.cc0, CcMetaVocab.ccbync4) val mainLicences: Map[Envri, URI] = Map( Envri.ICOS -> CpMetaVocab.ccby4, From 9a5b29439fae2ba9b35cdfef2db6017137d9c6a0 Mon Sep 17 00:00:00 2001 From: jonathanlindahl Date: Tue, 28 Oct 2025 11:17:55 +0100 Subject: [PATCH 2/5] fixed error that caused the new license to not get filtered out --- src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala b/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala index ad6350db..b7e3c3a3 100644 --- a/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala +++ b/src/main/scala/se/lu/nateko/cp/data/api/MetaVocab.scala @@ -66,5 +66,5 @@ object SitesMetaVocab extends MetaVocab(new URI("https://meta.fieldsites.se/onto object CcMetaVocab extends MetaVocab(new URI("https://creativecommons.org/")){ val cc0 = getRelative("publicdomain/zero/1.0/") - val ccbync4 = getRelative("licenses/by-nc/4.0") + val ccbync4 = getRelative("licenses/by-nc/4.0/") } From c2ad4d6f25a9b01ef984e29ae97df04108e37586 Mon Sep 17 00:00:00 2001 From: jonathanlindahl Date: Tue, 28 Oct 2025 13:26:30 +0100 Subject: [PATCH 3/5] simplified some validation as per pr suggestion --- .../se/lu/nateko/cp/data/services/upload/DownloadService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala index b976ceaa..d11b1e24 100644 --- a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala +++ b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala @@ -173,7 +173,7 @@ class DownloadService(coreConf: MetaCoreConfig, val upload: UploadService, val r val hdlProxy = if(dest.obj.doi.isDefined) prox.doi else prox.basic s"$hdlProxy$pid" } - val license = if (!dest.obj.references.licence.isEmpty) dest.obj.references.licence.get.name else "" + val license = dest.obj.references.licence.fold("")(_.name) s"$presense,${dest.fileName},${pidOpt.getOrElse("")},$landingPage,$license,$omissionReason\n" } Source(lines.map(ByteString.apply)) From a7d85e76eaf834188b09d4a56af8f13e9f82d592 Mon Sep 17 00:00:00 2001 From: jonathanlindahl Date: Wed, 29 Oct 2025 10:40:19 +0100 Subject: [PATCH 4/5] licencesToAccept explicitly checks for ccby4 --- .../services/upload/DownloadService.scala | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala index d11b1e24..822a0491 100644 --- a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala +++ b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala @@ -92,17 +92,19 @@ class DownloadService(coreConf: MetaCoreConfig, val upload: UploadService, val r } } - def licencesToAccept(hashes: Seq[Sha256Sum], uidOpt: Option[UserId])(using Envri): Future[Seq[URI]] = { - upload.meta.listLicences(hashes).flatMap{allLic => - val toAccept: Seq[URI] = allLic.distinct.filterNot(publicDomainLicences.contains) - Future.sequence( - toAccept.map{lic => - checkLicenceAcceptance(lic, uidOpt).map(lic -> _) + def licencesToAccept(hashes: Seq[Sha256Sum], uidOpt: Option[UserId])(using envri: Envri): Future[Seq[URI]] = { + mainLicences.get(envri) + .map { mainLicence => + upload.meta.listLicences(hashes).flatMap { allLicences => + if (allLicences.contains(mainLicence)) { + checkLicenceAcceptance(mainLicence, uidOpt).map { accepted => + if (accepted) Seq.empty else Seq(mainLicence) + } + } else { + Future.successful(Seq.empty) + } } - ).map{ - _.collect{case (lic, false) => lic} - } - } + }.getOrElse(Future.successful(Seq.empty)) } def inaccessibilityReason(dobj: StaticObject)(using Envri): Option[String] = From 77753e3ccabf03c1dc0ae0dd7e3ec2151f964fc3 Mon Sep 17 00:00:00 2001 From: jonathanlindahl Date: Wed, 29 Oct 2025 16:08:53 +0100 Subject: [PATCH 5/5] cleaned up string comparison --- .../cp/data/services/upload/DownloadService.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala index 822a0491..9a7d78f7 100644 --- a/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala +++ b/src/main/scala/se/lu/nateko/cp/data/services/upload/DownloadService.scala @@ -133,11 +133,11 @@ class DownloadService(coreConf: MetaCoreConfig, val upload: UploadService, val r .flatMapConcat{dests => val baseSource = Source.single(ZipEntry("!TOC.csv") -> destiniesToTocFileSource(dests)) - val shouldIncludeLincensePdf = dests.exists( - _.obj.references.licence.exists( - _.toString().contains("creativecommons.org/licenses/by/4.0") - ) - ) + val shouldIncludeLincensePdf = mainLicences.get(envri).exists { expectedUri => + dests.exists { d => + d.obj.references.licence.exists(_.url == expectedUri) + } + } if (shouldIncludeLincensePdf) { baseSource.concat(Source.single(ZipEntry("!LICENCE.pdf") -> licenceSource))