From 05b9239ea300352de4929fc906504bd6f7f758c6 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 26 Sep 2026 01:02:51 +0000 Subject: [PATCH 1/2] feat(doi): add guarded durable Zenodo publication lifecycle Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/doi.yml | 40 ++ Manifest.toml | 6 + Project.toml | 4 + .../src/components/AnalysisConfigEditor.tsx | 14 +- frontend/src/components/CladeCumulus.tsx | 1 + frontend/src/views/StudyView.tsx | 7 +- frontend/tsconfig.json | 9 +- src/MetaManifold.jl | 8 + src/analysis/AnalysisConfig.jl | 37 +- src/doi/AnalysisStore.jl | 92 +++++ src/doi/Bundles.jl | 248 ++++++++++++ src/doi/Publications.jl | 383 ++++++++++++++++++ src/doi/Storage.jl | 83 ++++ src/doi/Web.jl | 88 ++++ src/doi/Zenodo.jl | 193 +++++++++ src/doi/assets/publication.css | 44 ++ src/doi/assets/publication.js | 253 ++++++++++++ src/server/routes/analysis_config.jl | 113 ++---- src/server/routes/doi.jl | 221 ++++++++++ src/server/routes/studies.jl | 16 +- src/server/server.jl | 13 +- test/doi/Project.toml | 20 + test/doi/bootstrap.jl | 10 + test/doi/fixtures.jl | 117 ++++++ test/doi/runtests.jl | 4 + test/doi/tests.jl | 363 +++++++++++++++++ test/runtests.jl | 2 + test/unit/test_doi.jl | 35 ++ test/unit/test_doi_routes.jl | 111 +++++ 29 files changed, 2438 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/doi.yml create mode 100644 src/doi/AnalysisStore.jl create mode 100644 src/doi/Bundles.jl create mode 100644 src/doi/Publications.jl create mode 100644 src/doi/Storage.jl create mode 100644 src/doi/Web.jl create mode 100644 src/doi/Zenodo.jl create mode 100644 src/doi/assets/publication.css create mode 100644 src/doi/assets/publication.js create mode 100644 src/server/routes/doi.jl create mode 100644 test/doi/Project.toml create mode 100644 test/doi/bootstrap.jl create mode 100644 test/doi/fixtures.jl create mode 100644 test/doi/runtests.jl create mode 100644 test/doi/tests.jl create mode 100644 test/unit/test_doi.jl create mode 100644 test/unit/test_doi_routes.jl diff --git a/.github/workflows/doi.yml b/.github/workflows/doi.yml new file mode 100644 index 00000000..1d77149f --- /dev/null +++ b/.github/workflows/doi.yml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: MPL-2.0 +name: DOI publication contracts + +on: + pull_request: + paths: ['src/doi/**', 'src/server/**', 'src/analysis/AnalysisConfig.jl', 'test/doi/**', 'test/unit/test_doi*', 'scripts/link_doi.jl', 'config/schemas/doi*', 'config/templates/doi*', 'Project.toml', 'Manifest.toml', '.github/workflows/doi.yml'] + push: + branches: [main, 'arena/**'] + paths: ['src/doi/**', 'src/server/**', 'src/analysis/AnalysisConfig.jl', 'test/doi/**', 'test/unit/test_doi*', 'scripts/link_doi.jl', 'config/schemas/doi*', 'config/templates/doi*', 'Project.toml', 'Manifest.toml', '.github/workflows/doi.yml'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + contracts: + name: DOI contracts (no credentials or live deposits) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + JULIA_PKG_PRECOMPILE_AUTO: '0' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3.0.2 + with: + version: '1.12.5' + - uses: julia-actions/cache@a7bed9df697e5d7309d68afe7542a87621a8b6c8 # v3.3.0 + - name: Syntax and source dependency contracts + run: julia --startup-file=no config/ci/lint_source.jl + - name: Instantiate isolated HTTP-only test environment + run: julia --project=test/doi -e 'using Pkg; Pkg.instantiate()' + - name: Publication lifecycle, security, restart and failure contracts + run: julia --project=test/doi test/doi/runtests.jl + - name: Preserve resolved test environment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: doi-test-environment + path: test/doi/Manifest.toml + if-no-files-found: ignore diff --git a/Manifest.toml b/Manifest.toml index dbfb54e6..7fb42924 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -409,6 +409,12 @@ deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" version = "1.11.0" +[[deps.MD5]] +deps = ["Random", "SHA"] +git-tree-sha1 = "1576f756617d31eb397a4a517b68562fd28dc2b4" +uuid = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" +version = "0.2.3" + [[deps.MbedTLS]] deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] git-tree-sha1 = "8785729fa736197687541f7053f6d8ab7fc44f92" diff --git a/Project.toml b/Project.toml index 6ec20689..1878f08e 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ DuckDB = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" @@ -27,6 +28,9 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" BenchmarkTools = "1.8.0" CSV = "0.10" DataFrames = "1.8" +MD5 = "0.2.3" +HTTP = "1" +JSON3 = "1" PackageCompiler = "2.2.5" RCall = "0.14" YAML = "0.4" diff --git a/frontend/src/components/AnalysisConfigEditor.tsx b/frontend/src/components/AnalysisConfigEditor.tsx index 4cb7393d..aec0e311 100644 --- a/frontend/src/components/AnalysisConfigEditor.tsx +++ b/frontend/src/components/AnalysisConfigEditor.tsx @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-only import { useState } from 'react' +import { apiUrl } from '../api/client' import type { AnalysisConfig, ValidationError } from '../types/analysis_config' import { contextHelp, isDangerous, DANGER_ACK_TOKEN } from '../types/analysis_config' import { DangerBanner } from './DangerBanner' @@ -7,6 +8,8 @@ import { AdvancedAnalysisExpander } from './AdvancedAnalysisExpander' interface AnalysisConfigEditorProps { evidenceMode: boolean + /** Thin launch link: publication UI and lifecycle remain Julia-owned. */ + study?: string config: AnalysisConfig onChange: (config: AnalysisConfig) => void onSave: () => void @@ -14,7 +17,7 @@ interface AnalysisConfigEditorProps { validationErrors?: ValidationError[] } -export function AnalysisConfigEditor({ evidenceMode, config, onChange, onSave, availableMetadataColumns, validationErrors }: AnalysisConfigEditorProps) { +export function AnalysisConfigEditor({ evidenceMode, study, config, onChange, onSave, availableMetadataColumns, validationErrors }: AnalysisConfigEditorProps) { const [helpField, setHelpField] = useState(null) const [showJson, setShowJson] = useState(false) @@ -310,6 +313,15 @@ export function AnalysisConfigEditor({ evidenceMode, config, onChange, onSave, a DOI-ready: Bundle includes JSON + Nickel + DEED + DataCite + provenance. + {evidenceMode && study && ( +

+ + Mint DOI / view publication badge… + + {' '}Opens the Julia publication screen for a saved configuration. Preparing a draft does not publish it. +

+ )} + {/* Actions */}
+

+ Evidence & DOI publication + {' '}— review an immutable analysis bundle and publish it on Zenodo. +

+ {loading && } {error &&

{error}

} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 89efbfc4..542fdd3c 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -29,12 +29,11 @@ "noFallthroughCasesInSwitch": true, "forceConsistentCasingInFileNames": true, "verbatimModuleSyntax": true, - "baseUrl": ".", "paths": { - "@api/*": ["src/api/*"], - "@components/*": ["src/components/*"], - "@views/*": ["src/views/*"], - "@hooks/*": ["src/hooks/*"] + "@api/*": ["./src/api/*"], + "@components/*": ["./src/components/*"], + "@views/*": ["./src/views/*"], + "@hooks/*": ["./src/hooks/*"] } }, "include": ["src"] diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index e8c4b006..afbd8fff 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -27,6 +27,13 @@ include("pipeline/merge_taxa.jl") include("pipeline/dada2.jl") include("pipeline/swarm.jl") +# DOI infrastructure is independent of the scientific/R runtime. +include("doi/Storage.jl") +include("doi/Zenodo.jl") +include("doi/Bundles.jl") +include("doi/Publications.jl") +include("doi/Web.jl") + # Analysis include("analysis/numeric_policy.jl") # Exact summaries are catalogue item 1 and are built on the numeric policy, so they follow it. @@ -34,6 +41,7 @@ include("analysis/exact_summaries.jl") include("analysis/diversity.jl") include("analysis/analysis.jl") include("analysis/AnalysisConfig.jl") +include("doi/AnalysisStore.jl") include("analysis/clade_cumulus.jl") # Estimation fits what AnalysisConfig declares and Execution runs it, so it sits between them. include("analysis/estimation.jl") diff --git a/src/analysis/AnalysisConfig.jl b/src/analysis/AnalysisConfig.jl index 852d3293..f34bb31c 100644 --- a/src/analysis/AnalysisConfig.jl +++ b/src/analysis/AnalysisConfig.jl @@ -32,6 +32,7 @@ using UUIDs using JSON3 using OrderedCollections using Logging +using ..DOIBundles: write_checksums! # Re-use provenance from core using ..Provenance: CapturedEnvironment, probe_metamanifold, probe_host @@ -1548,13 +1549,19 @@ end """ create_doi_bundle(config, [result]; output_dir, authors, title, license, description) -Create or update `output_dir` with DataCite metadata, JSON, Nickel, DEED, -provenance, and content-hash files, then return the directory path. An optional -result is embedded in the DataCite document. Dangerous configurations also write -`DANGER_BANNER.txt` and emit a warning; existing files with the same names are -overwritten. +Create a fresh `output_dir` with DataCite metadata, JSON, Nickel, DEED, +provenance, and per-file SHA-256 checksums, then return the directory path. +A supplied result must belong to the exact config. Non-empty destinations are +refused: stale results or DANGER banners must never enter a new publication. +Dangerous configurations also write `DANGER_BANNER.txt` and emit a warning. """ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult,Nothing}=nothing; output_dir::String="doi_bundle_$(config.id)", authors::Vector{String}=String[], title::String="MetaManifold Analysis Bundle", license::String="CC-BY-4.0", description::String="Differential abundance analysis") + islink(output_dir) && throw(ArgumentError("DOI bundle directory must not be a symlink")) + isdir(output_dir) && !isempty(readdir(output_dir)) && throw(ArgumentError("DOI bundle requires an empty destination")) + if !isnothing(result) + result.config_id == config.id && result.config_hash == config.hash && result.method == config.method || + throw(ArgumentError("DOI bundle result must belong to the exact configuration")) + end mkpath(output_dir) # DataCite JSON @@ -1566,7 +1573,7 @@ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult, "descriptions" => [OrderedDict("description" => description, "descriptionType" => "Abstract")], "publicationYear" => year(config.created_at), "publisher" => "MetaManifold-WebUI", - "resourceType" => OrderedDict("resourceTypeGeneral" => "Dataset", "resourceType" => "AnalysisConfig"), + "types" => OrderedDict("resourceTypeGeneral" => "Dataset", "resourceType" => isnothing(result) ? "Analysis configuration (no results)" : "Analysis results"), "subjects" => [ OrderedDict("subject" => "microbiome"), OrderedDict("subject" => "differential abundance"), @@ -1577,8 +1584,8 @@ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult, "version" => config.schema_version, "rightsList" => [OrderedDict("rights" => license)], "dates" => [OrderedDict("date" => string(config.created_at), "dateType" => "Created")], - "relatedIdentifiers" => [ - OrderedDict("relatedIdentifier" => config.hash, "relatedIdentifierType" => "SHA256", "relationType" => "IsIdenticalTo"), + "alternateIdentifiers" => [ + OrderedDict("alternateIdentifier" => config.hash, "alternateIdentifierType" => "SHA-256"), ], "schemaVersion" => "http://datacite.org/schema/kernel-4", "config" => JSON3.read(to_json(config)), @@ -1601,9 +1608,8 @@ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult, "method" => METHOD_TO_STRING[result.method], "results" => result.results ) - datacite["relatedIdentifiers"] = vcat(datacite["relatedIdentifiers"], [ - OrderedDict("relatedIdentifier" => result.hash, "relatedIdentifierType" => "SHA256", "relationType" => "HasPart") - ]) + push!(datacite["alternateIdentifiers"], + OrderedDict("alternateIdentifier" => result.hash, "alternateIdentifierType" => "SHA-256")) end # Write files @@ -1675,9 +1681,10 @@ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult, | `analysis_config.json` | Machine-readable analysis configuration | | `analysis_config.ncl` | Nickel serialisation of the configuration | | `analysis_config_chora.deed` | DEED attestation of the configuration | - | `analysis_result.json` | The analysis result this bundle was minted for | + | `analysis_result.json` | Selected result, only when explicitly included | | `provenance.json` | Captured software and host environment | | `content_hash.txt` | Content hash of the configuration | + | `checksums.sha256` | SHA-256 of every payload file | ## Authors @@ -1685,6 +1692,10 @@ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult, ## Reproducibility + **Payload:** $(isnothing(result) ? "Configuration only — no analysis results are included." : "Configuration and the explicitly selected result $(result.id).") + No DOI has been minted by this local export. Publication is a separate, + explicitly confirmed operation. Raw inputs and reference databases are not included. + The configuration is immutable and content-hashed. Re-running the analysis requires the same MetaManifold version and database snapshot recorded in `provenance.json`. @@ -1695,6 +1706,8 @@ function create_doi_bundle(config::AnalysisConfig, result::Union{AnalysisResult, write(io, config.hash) end + write_checksums!(output_dir) + @info "Created DOI-ready bundle" output_dir config_id=config.id hash=config.hash dangerous=config.dangerous return output_dir diff --git a/src/doi/AnalysisStore.jl b/src/doi/AnalysisStore.jl new file mode 100644 index 00000000..3d375193 --- /dev/null +++ b/src/doi/AnalysisStore.jl @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: MPL-2.0 +# Persistence required by the DOI workflow: a backend restart must not erase the +# config/result named by a reviewed draft. The publication journal keeps its own +# immutable snapshots as well, so it never follows a moving "latest result". +module AnalysisStore + +using JSON3, Dates, UUIDs, OrderedCollections +using ..DOIStorage +import ..AnalysisConfig + +export configs, results, save_config!, save_result!, delete_config!, result_json + +function _record_path(root, kind, id) + try UUID(id) catch; throw(PublicationError(400, "invalid_id", "Analysis ID must be a UUID.")) end + joinpath(root, kind, id * ".json") +end + +function configs(root) + directory = joinpath(root, "configs") + output = Dict{String,AnalysisConfig.AnalysisConfigStruct}() + isdir(directory) || return output + for name in readdir(directory) + endswith(name, ".json") || continue + path = joinpath(directory, name) + islink(path) && throw(PublicationError(409, "unsafe_storage", "Stored analysis must not be a symlink.")) + text = read(path, String) + cfg = AnalysisConfig.from_json(text) + # from_json supports caller-supplied hashes for legacy uses; storage must + # not trust one. Reconstruct without it and compare the computed hash. + data = JSON3.read(text, Dict{String,Any}) + data["hash"] = nothing + AnalysisConfig.from_json(JSON3.write(data)).hash == cfg.hash || + throw(PublicationError(409, "config_hash_mismatch", "Stored analysis configuration failed its content-hash check.")) + name == cfg.id * ".json" || throw(PublicationError(409, "config_id_mismatch", "Stored configuration ID differs from its filename.")) + output[cfg.id] = cfg + end + return output +end + +function result_json(result::AnalysisConfig.AnalysisResult) + JSON3.write(OrderedDict{String,Any}("id" => result.id, "config_id" => result.config_id, + "config_hash" => result.config_hash, "created_at" => string(result.created_at), + "method" => AnalysisConfig.METHOD_TO_STRING[result.method], "results" => result.results, + "provenance" => result.provenance, "hash" => result.hash)) +end + +function results(root) + directory = joinpath(root, "results") + output = Dict{String,AnalysisConfig.AnalysisResult}() + isdir(directory) || return output + for name in readdir(directory) + endswith(name, ".json") || continue + path = joinpath(directory, name) + islink(path) && throw(PublicationError(409, "unsafe_storage", "Stored result must not be a symlink.")) + data = JSON3.read(read(path, String)) + method = AnalysisConfig.METHOD_STRINGS[String(data.method)] + # Keep result entry ordering: the existing scientific hash format is + # order-sensitive. Do not canonical-sort the original scientific object. + result = AnalysisConfig.AnalysisResult(id=String(data.id), config_id=String(data.config_id), + config_hash=String(data.config_hash), created_at=DateTime(data.created_at), method=method, + results=OrderedDict{String,Any}(String(k) => v for (k, v) in data.results), + provenance=OrderedDict{String,Any}(String(k) => v for (k, v) in data.provenance)) + result.hash == data.hash && name == result.id * ".json" || + throw(PublicationError(409, "result_hash_mismatch", "Stored analysis result failed its content-hash check.")) + output[result.id] = result + end + return output +end + +function _save(root, kind, id, text) + with_publication_lock(root) do + path = _record_path(root, kind, id) + if isfile(path) + read(path, String) == text || throw(PublicationError(409, "immutable_analysis", "An immutable analysis ID already exists with different content.")) + else + atomic_write(path, text) + end + return id + end +end +save_config!(root, cfg::AnalysisConfig.AnalysisConfigStruct) = _save(root, "configs", cfg.id, AnalysisConfig.to_json(cfg)) +save_result!(root, result::AnalysisConfig.AnalysisResult) = _save(root, "results", result.id, result_json(result)) + +function delete_config!(root, id) + with_publication_lock(root) do + path = _record_path(root, "configs", id) + isfile(path) || throw(PublicationError(404, "config_not_found", "Analysis configuration not found.")) + rm(path) + end +end + +end # module AnalysisStore diff --git a/src/doi/Bundles.jl b/src/doi/Bundles.jl new file mode 100644 index 00000000..3c1cff1f --- /dev/null +++ b/src/doi/Bundles.jl @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: MPL-2.0 +module DOIBundles + +using JSON3, SHA, MD5, Dates, UUIDs +using ..DOIStorage: PublicationError, canonical_json, file_sha256, atomic_write + +export validate_metadata, snapshot, write_checksums!, verify_checksums, archive_bundle, + decorate_bundle!, zenodo_metadata, file_md5, citation_cff, citation_text, + BUNDLE_FILES, LICENSES + +const LICENSES = Dict("CC-BY-4.0" => "cc-by-4.0", "CC-BY-SA-4.0" => "cc-by-sa-4.0", "CC0-1.0" => "cc0-1.0") +const BASE_FILES = Set(["analysis_config.json", "analysis_config.ncl", "analysis_config_chora.deed", + "datacite.json", "provenance.json", "content_hash.txt", "README.md"]) +const BUNDLE_FILES = union(BASE_FILES, Set(["analysis_result.json", "DANGER_BANNER.txt", "checksums.sha256"])) +const DECORATED_FILES = union(BUNDLE_FILES, Set(["publication.json", "CITATION.cff", "CITATION.txt", "zenodo.json"])) +fail(message) = throw(PublicationError(422, "invalid_publication", message)) +file_md5(path::AbstractString) = open(io -> bytes2hex(md5(io)), path) + +function _text(value, field; max_length=500, optional=false) + value isa AbstractString || fail("$field must be text.") + text = strip(String(value)) + (!optional && isempty(text)) && fail("$field must not be empty.") + length(text) <= max_length || fail("$field is too long.") + any(c -> iscntrl(c) && c != '\n' && c != '\t', text) && fail("$field contains control characters.") + return text +end + +function _github_url(value, field, kind) + text = _text(value, field; max_length=2048, optional=true) + isempty(text) && return nothing + pattern = kind == :release ? r"^https://github\.com/[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+/releases/tag/[A-Za-z0-9_.~%+/-]+$" : + r"^https://github\.com/(users|orgs)/[A-Za-z0-9][A-Za-z0-9-]*/projects/[1-9][0-9]*$" + occursin(pattern, text) || fail("$field must be a public GitHub $(kind == :release ? "release tag" : "Projects v2") URL without credentials, query, or fragment.") + # Refuse URL parser normalisation and malformed percent escapes. + any(s -> s in (".", ".."), split(text, '/')) && fail("$field contains a traversal segment.") + occursin(r"%(?![0-9A-Fa-f]{2})", text) && fail("$field contains an invalid escape.") + occursin(r"(?i)%2e|%2f|%5c|%0[0-9a-f]|%1[0-9a-f]|%7f", text) && fail("$field contains an unsafe encoded path.") + return text +end + +function _orcid(value) + text = _text(value, "creator.orcid"; max_length=19) + occursin(r"^[0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9]{3}[0-9X]$", text) || fail("ORCID must use the 0000-0000-0000-000X format.") + digits = replace(text, "-" => "") + total = 0 + for d in digits[1:15] + total = (total + (Int(d) - Int('0'))) * 2 + end + check = (12 - total % 11) % 11 + string(last(digits)) == (check == 10 ? "X" : string(check)) || fail("ORCID checksum is invalid.") + return text +end + +"""Validate an explicit publication request, independently of analysis-schema defaults.""" +function validate_metadata(input::AbstractDict; today=Date(now(UTC))) + allowed = Set(["title", "description", "creators", "license", "publication_date", "version", "github_release_url", "github_project_url"]) + all(k -> k in allowed, keys(input)) || fail("Unknown publication metadata field. Tokens, API URLs and DOI overrides are never accepted.") + title = _text(get(input, "title", nothing), "title"; max_length=250) + description = _text(get(input, "description", nothing), "description"; max_length=10000) + creators = get(input, "creators", nothing) + creators isa AbstractVector && 1 <= length(creators) <= 100 || fail("Provide 1–100 named creators.") + names = Dict{String,String}[] + for creator in creators + creator isa AbstractDict || fail("Each creator must be an object.") + all(k -> k in ("name", "affiliation", "orcid"), keys(creator)) || fail("Unknown creator field.") + name = _text(get(creator, "name", nothing), "creator.name"; max_length=250) + lowercase(name) in ("anonymous", "unknown", "test", "your name") && fail("Replace placeholder creators with the actual authors before publication.") + record = Dict("name" => name) + if haskey(creator, "affiliation") + record["affiliation"] = _text(creator["affiliation"], "creator.affiliation"; optional=true) + end + haskey(creator, "orcid") && (record["orcid"] = _orcid(creator["orcid"])) + push!(names, record) + end + license = get(input, "license", nothing) + license isa AbstractString && haskey(LICENSES, license) || fail("Choose an explicit supported licence: CC-BY-4.0, CC-BY-SA-4.0 or CC0-1.0.") + date_string = _text(get(input, "publication_date", string(today)), "publication_date"; max_length=10) + date = try Date(date_string, dateformat"yyyy-mm-dd") catch; fail("publication_date must be YYYY-MM-DD.") end + string(date) == date_string && Date(1900) <= date <= today || fail("publication_date must be a real date between 1900 and today.") + version = _text(get(input, "version", "1.0.0"), "version"; max_length=100) + result = Dict{String,Any}("title" => title, "description" => description, "creators" => names, + "license" => String(license), "publication_date" => date_string, "version" => version) + for (field, kind) in (("github_release_url", :release), ("github_project_url", :project)) + value = get(input, field, nothing) + result[field] = isnothing(value) ? nothing : _github_url(value, field, kind) + end + return result +end + +function _regular_files(directory, allowed) + isdir(directory) && !islink(directory) || fail("Bundle must be a regular directory.") + names = readdir(directory) + all(name -> name in allowed && isfile(joinpath(directory, name)) && !islink(joinpath(directory, name)), names) || + fail("Bundle contains an unexpected entry, directory or symlink. Only the documented bundle files may be published.") + return names +end + +"""Read and bind the exact config/result bytes; refuse scaffolds and mismatched results.""" +function snapshot(directory::String) + names = _regular_files(directory, BUNDLE_FILES) + issubset(BASE_FILES, Set(names)) || fail("Bundle is missing required config, provenance or metadata files.") + isfile(joinpath(directory, "checksums.sha256")) && verify_checksums(directory) + config_text = read(joinpath(directory, "analysis_config.json"), String) + cfg = try JSON3.read(config_text, Dict{String,Any}) catch; fail("Invalid analysis_config.json.") end + try UUID(cfg["id"]) catch; fail("Bundle config ID must be a UUID.") end + hash = get(cfg, "hash", "") + hash isa AbstractString && occursin(r"^[0-9a-f]{64}$", hash) || fail("Bundle config hash must be SHA-256.") + strip(read(joinpath(directory, "content_hash.txt"), String)) == hash || fail("Bundle content_hash.txt does not match its configuration.") + cfg["dangerous"] isa Bool || fail("Bundle dangerous flag must be boolean.") + cfg["dangerous"] && !("DANGER_BANNER.txt" in names) && fail("A dangerous configuration must retain its DANGER banner.") + result = nothing + result_sha = nothing + if "analysis_result.json" in names + text = read(joinpath(directory, "analysis_result.json"), String) + result = try JSON3.read(text, Dict{String,Any}) catch; fail("Invalid analysis_result.json.") end + get(result, "config_id", nothing) == cfg["id"] && get(result, "config_hash", nothing) == hash && get(result, "method", nothing) == cfg["method"] || + fail("Result does not belong to this exact analysis configuration.") + try UUID(result["id"]) catch; fail("Result ID must be a UUID.") end + rhash = get(result, "hash", "") + rhash isa AbstractString && occursin(r"^[0-9a-f]{64}$", rhash) || fail("Result hash must be SHA-256.") + payload = get(result, "results", nothing) + payload isa AbstractDict && !isempty(payload) || fail("Empty/scaffold results cannot be published. Select a configuration-only bundle explicitly instead.") + prov = get(result, "provenance", Dict()) + get(prov, "mock", false) === true && fail("Mock results cannot be published.") + startswith(lowercase(string(get(prov, "note", ""))), "mock result") && fail("Mock results cannot be published.") + estimation = get(prov, "estimation", Dict()) + get(estimation, "status", "") in ("not_run", "failed") && fail("An analysis which did not run cannot be published as a result.") + result_sha = bytes2hex(sha256(text)) + end + return Dict{String,Any}("config_id" => cfg["id"], "config_hash" => hash, + "config_file_sha256" => bytes2hex(sha256(config_text)), "dangerous" => cfg["dangerous"], + "result_id" => isnothing(result) ? nothing : result["id"], + "result_hash" => isnothing(result) ? nothing : result["hash"], "result_file_sha256" => result_sha, + "kind" => isnothing(result) ? "configuration" : "analysis_result") +end + +function write_checksums!(directory::String) + names = sort!(_regular_files(directory, DECORATED_FILES)) + text = join((file_sha256(joinpath(directory, name)) * " " * name * "\n" for name in names if name != "checksums.sha256")) + write(joinpath(directory, "checksums.sha256"), text) + return file_sha256(joinpath(directory, "checksums.sha256")) +end + +function verify_checksums(directory::String) + names = _regular_files(directory, DECORATED_FILES) + path = joinpath(directory, "checksums.sha256") + isfile(path) || fail("Bundle checksum manifest is missing.") + seen = Set{String}() + for line in eachline(path) + m = match(r"^([0-9a-f]{64}) ([A-Za-z0-9_.-]+)$", line) + !isnothing(m) || fail("Invalid bundle checksum manifest.") + hash, name = m.captures + name in names && name != "checksums.sha256" && !(name in seen) || fail("Checksum manifest contains an unexpected or repeated file.") + file_sha256(joinpath(directory, name)) == hash || fail("Bundle integrity check failed; a file changed after preparation.") + push!(seen, name) + end + seen == setdiff(Set(names), Set(["checksums.sha256"])) || fail("Bundle checksum manifest does not cover every file.") + return true +end + +function archive_bundle(directory::String, destination::String) + verify_checksums(directory) + isnothing(Sys.which("zip")) && throw(PublicationError(503, "zip_unavailable", "The server needs the zip executable to create a DOI bundle.")) + ispath(destination) && fail("Refusing to overwrite an existing publication archive.") + # Relative, sorted members; never archive absolute temporary paths or stale files. + files = sort!(readdir(directory)) + try + run(Cmd(`zip -q -X $(abspath(destination)) $files`; dir=directory)) + catch + isfile(destination) && rm(destination) + throw(PublicationError(500, "archive_failed", "Could not create the DOI archive. No upload was attempted.")) + end + return destination +end + +_html(text) = replace(text, '&' => "&", '<' => "<", '>' => ">", '"' => """, '\'' => "'") + +function zenodo_metadata(metadata, binding, publication_id) + kind = binding["kind"] == "configuration" ? "Configuration only; no analysis results are included." : "Analysis configuration, selected results and provenance." + warning = binding["dangerous"] ? " DANGER: scientific overrides are present; see DANGER_BANNER.txt and disclose them when citing." : "" + related = Dict{String,String}[] + for (field, relation) in (("github_release_url", "isSupplementTo"), ("github_project_url", "references")) + isnothing(metadata[field]) || push!(related, Dict("identifier" => metadata[field], "relation" => relation)) + end + Dict{String,Any}("title" => metadata["title"], "upload_type" => "dataset", + "description" => "

" * _html(metadata["description"]) * "

" * kind * warning * "

", + "creators" => metadata["creators"], "access_right" => "open", "license" => LICENSES[metadata["license"]], + "publication_date" => metadata["publication_date"], "version" => metadata["version"], + "prereserve_doi" => true, "related_identifiers" => related, + "notes" => "MetaManifold publication " * publication_id * "; config SHA-256 " * binding["config_hash"], + "keywords" => ["MetaManifold", "reproducibility", binding["kind"]]) +end + +function citation_text(metadata, doi) + join((c["name"] for c in metadata["creators"]), "; ") * " (" * metadata["publication_date"][1:4] * "). " * + metadata["title"] * " (" * metadata["version"] * ") [Data set]. Zenodo. https://doi.org/" * doi +end + +function citation_cff(metadata, doi) + io = IOBuffer() + println(io, "cff-version: 1.2.0\nmessage: \"Cite the published Zenodo record; reserved and sandbox DOIs are not production citations.\"\ntype: dataset") + for (key, value) in (("title", metadata["title"]), ("doi", doi), ("version", metadata["version"]), + ("date-released", metadata["publication_date"]), ("license", metadata["license"])) + println(io, key, ": ", JSON3.write(value)) # JSON strings are valid quoted YAML scalars. + end + println(io, "authors:") + for creator in metadata["creators"] + println(io, " - name: ", JSON3.write(creator["name"])) + haskey(creator, "orcid") && println(io, " orcid: ", JSON3.write("https://orcid.org/" * creator["orcid"])) + end + return String(take!(io)) +end + +function decorate_bundle!(directory, metadata, binding, publication_id, environment, deposit_id, doi) + # Publication is an external attestation, not an edit to the hashed config/result. + publication = merge(copy(binding), Dict("schema_version" => "1.0.0", "publication_id" => publication_id, + "environment" => environment, "deposition_id" => deposit_id, "doi" => doi, "state" => "reserved", + "github_release_url" => metadata["github_release_url"], "github_project_url" => metadata["github_project_url"])) + write(joinpath(directory, "publication.json"), canonical_json(publication)) + write(joinpath(directory, "CITATION.cff"), citation_cff(metadata, doi)) + write(joinpath(directory, "CITATION.txt"), citation_text(metadata, doi) * "\n") + write(joinpath(directory, "zenodo.json"), canonical_json(zenodo_metadata(metadata, binding, publication_id))) + # DataCite metadata uses its own schema, NOT the deposition request schema. + related = [Dict("relatedIdentifier" => metadata[f], "relatedIdentifierType" => "URL", "relationType" => relation) + for (f, relation) in (("github_release_url", "IsSupplementTo"), ("github_project_url", "References")) if !isnothing(metadata[f])] + datacite = Dict("identifiers" => [Dict("identifier" => doi, "identifierType" => "DOI")], + "creators" => [Dict("name" => c["name"]) for c in metadata["creators"]], + "titles" => [Dict("title" => metadata["title"])], "publisher" => "Zenodo", + "publicationYear" => parse(Int, metadata["publication_date"][1:4]), + "types" => Dict("resourceTypeGeneral" => "Dataset", "resourceType" => binding["kind"]), + "descriptions" => [Dict("description" => metadata["description"], "descriptionType" => "Abstract")], + "rightsList" => [Dict("rights" => metadata["license"], "rightsIdentifier" => lowercase(metadata["license"]), "rightsIdentifierScheme" => "SPDX")], + "relatedIdentifiers" => related, "version" => metadata["version"], + "alternateIdentifiers" => [Dict("alternateIdentifier" => binding["config_hash"], "alternateIdentifierType" => "SHA-256")]) + write(joinpath(directory, "datacite.json"), canonical_json(datacite)) + open(joinpath(directory, "README.md"), "a") do io + println(io, "\n## Zenodo publication\n\nEnvironment: ", environment, ". Kind: ", binding["kind"], ".") + println(io, "\nReserved DOI: ", doi, ". It is registered only after publication. Sandbox records are test records.") + println(io, "\n", citation_text(metadata, doi)) + println(io, "\n`publication.json` binds the DOI to the original config/result hashes without changing them.") + println(io, "`checksums.sha256` covers every payload file. The separate post-publication receipt records the archive hash (avoiding a circular hash).") + println(io, "Raw inputs and reference databases are not included; their availability must be described by the authors.") + end + write_checksums!(directory) + return directory +end + +end # module DOIBundles diff --git a/src/doi/Publications.jl b/src/doi/Publications.jl new file mode 100644 index 00000000..7226088f --- /dev/null +++ b/src/doi/Publications.jl @@ -0,0 +1,383 @@ +# SPDX-License-Identifier: MPL-2.0 +# A durable, recoverable two-phase publication journal. No tokens in state or bundles. +module DOIPublications + +using JSON3, SHA, Dates +using ..DOIStorage +using ..DOIBundles +import ..Zenodo + +export prepare!, resume!, publish!, refresh!, recover_creation!, status, publications, + publication_path, confirmation_phrase, receipt, download_path + +const SCHEMA_VERSION = "1.0.0" +const MAX_ARCHIVE_BYTES = 50_000_000_000 +const STATES = Set(["preparing", "creating", "creation_uncertain", "draft", "ready", "publishing", "publication_uncertain", "published"]) + +function publication_path(root::AbstractString, id::AbstractString) + occursin(r"^[0-9a-f]{64}$", id) || throw(PublicationError(404, "publication_not_found", "Publication not found.")) + joinpath(root, id) +end +_state_path(root, id) = joinpath(publication_path(root, id), "state.json") +_archive_name(id) = "metamanifold-" * id * ".zip" +_archive(root, s) = joinpath(publication_path(root, s["id"]), _archive_name(s["id"])) + +function _read(root, id) + path = _state_path(root, id) + isfile(path) && !islink(path) || throw(PublicationError(404, "publication_not_found", "Publication not found.")) + s = try read_json(path) catch; throw(PublicationError(409, "corrupt_publication", "Publication state cannot be read. Restore its journal from backup; do not create a replacement.")) end + get(s, "schema_version", nothing) == SCHEMA_VERSION && get(s, "id", nothing) == id && get(s, "state", "") in STATES || + throw(PublicationError(409, "corrupt_publication", "Unsupported or inconsistent publication journal.")) + return s +end + +function _save(root, s, event) + s["updated_at"] = string(now(UTC)) * "Z" + push!(s["events"], Dict("at" => s["updated_at"], "event" => event, "state" => s["state"])) + atomic_json(_state_path(root, s["id"]), s) + return s +end + +function _client_matches(s, client) + s["environment"] == client.environment || throw(PublicationError(409, "environment_mismatch", "This publication belongs to a different Zenodo environment. Ask the operator to select its original environment.")) +end + +confirmation_phrase(s) = "PUBLISH " * s["environment"] * " " * something(s["deposition_id"], "unprepared") + +function _public(s) + # Deliberate allowlist: internal paths, raw remote responses and credentials + # cannot accidentally become a future public API field. + fields = ("schema_version", "id", "state", "environment", "binding", "metadata", "deposition_id", + "reserved_doi", "doi", "record_url", "bundle_sha256", "bundle_md5", "bundle_size", + "created_at", "updated_at", "published_at", "last_error") + result = Dict{String,Any}(key => get(s, key, nothing) for key in fields) + result["confirmation_phrase"] = confirmation_phrase(s) + result["citation"] = s["state"] == "published" ? citation_text(s["metadata"], s["doi"]) : nothing + result["doi_url"] = s["state"] == "published" ? "https://doi.org/" * s["doi"] : nothing + result["draft_url"] = isnothing(s["deposition_id"]) ? nothing : Zenodo.ORIGINS[s["environment"]] * "/deposit/" * s["deposition_id"] + result["test_record"] = s["environment"] == "sandbox" + return result +end +status(root, id) = _public(_read(root, id)) + +function publications(root) + isdir(root) || return Dict{String,Any}[] + [_public(_read(root, id)) for id in sort!(readdir(root)) if occursin(r"^[0-9a-f]{64}$", id) && isfile(_state_path(root, id))] +end + +function _local_integrity(root, s) + archive = _archive(root, s) + isfile(archive) && !islink(archive) && file_sha256(archive) == s["bundle_sha256"] && filesize(archive) == s["bundle_size"] || + throw(PublicationError(409, "bundle_changed", "The frozen archive is missing or changed. Restore its exact bytes; do not publish a replacement under this confirmation.")) + return archive +end + +# Zenodo adds fields to creators/related identifiers; compare the fields we sent, +# not whole response dictionaries. Arrays remain order-sensitive for authorship. +_subset(expected::AbstractDict, actual::AbstractDict) = all(haskey(actual, k) && _subset(v, actual[k]) for (k, v) in expected) +_subset(expected::AbstractVector, actual::AbstractVector) = length(expected) == length(actual) && all(_subset(a, b) for (a, b) in zip(expected, actual)) +_subset(expected, actual) = expected == actual + +function _remote_metadata(s, deposit, client) + Zenodo.checked_id(get(deposit, "id", nothing)) == s["deposition_id"] || + throw(PublicationError(409, "deposition_mismatch", "Zenodo returned a different deposition. Refusing to continue.")) + expected = zenodo_metadata(s["metadata"], s["binding"], s["id"]) + delete!(expected, "prereserve_doi") + actual = get(deposit, "metadata", nothing) + actual isa AbstractDict && _subset(expected, actual) || + throw(PublicationError(409, "remote_metadata_changed", "Zenodo metadata differs from the frozen publication request. Review the existing draft; it was not overwritten or published.")) + doi = get(deposit, "submitted", false) === true ? + Zenodo.checked_doi(client, get(deposit, "doi", get(actual, "doi", nothing))) : Zenodo.reserved_doi(client, deposit) + if !isnothing(s["reserved_doi"]) + doi == s["reserved_doi"] || throw(PublicationError(409, "doi_changed", "The Zenodo DOI differs from the reserved identifier.")) + end + return doi +end + +function _file_matches(file, s) + file isa AbstractDict || return false + name = get(file, "filename", get(file, "name", get(file, "key", nothing))) + name == _archive_name(s["id"]) || return false + checksum = get(file, "checksum", nothing) + checksum isa AbstractString || return false + replace(lowercase(checksum), r"^md5:" => "") == s["bundle_md5"] || return false + rawsize = get(file, "filesize", get(file, "size", nothing)) + size = rawsize isa AbstractString ? tryparse(Int, rawsize) : rawsize + size isa Integer && !(size isa Bool) && size == s["bundle_size"] +end + +function _remote_files(s, deposit) + files = get(deposit, "files", nothing) + files isa AbstractVector && length(files) == 1 && _file_matches(only(files), s) || + throw(PublicationError(409, "remote_files_changed", "Zenodo must contain exactly the reviewed archive with the matching size and MD5 checksum. No publication was attempted.")) + return true +end + +function _make_archive!(root, s) + directory = publication_path(root, s["id"]) + archive = _archive(root, s) + if !isnothing(s["bundle_sha256"]) + _local_integrity(root, s) + return + end + payload = joinpath(directory, "payload") + # Before an archive hash is journalled no upload can have started. Interrupted + # local builds may be restarted, but a journalled archive is never regenerated. + isdir(payload) && rm(payload; recursive=true) + isfile(archive) && rm(archive) + cp(joinpath(directory, "source"), payload) + decorate_bundle!(payload, s["metadata"], s["binding"], s["id"], s["environment"], s["deposition_id"], s["reserved_doi"]) + archive_bundle(payload, archive) + chmod(archive, 0o600) + filesize(archive) <= MAX_ARCHIVE_BYTES || throw(PublicationError(422, "bundle_too_large", "The archive exceeds Zenodo's 50 GB record limit.")) + s["bundle_sha256"] = file_sha256(archive) + s["bundle_md5"] = file_md5(archive) # Protocol integrity only; SHA-256 is the provenance identity. + s["bundle_size"] = filesize(archive) + _save(root, s, "archive_frozen") +end + +function _remember_error!(root, s, e) + # Only our sanitised exceptions may reach the journal. Never persist raw HTTP + # errors, arbitrary exception strings, remote metadata, or token-bearing URLs. + s["last_error"] = e isa Zenodo.RemoteError || e isa PublicationError ? Dict("code" => e.code, "message" => e.message) : + Dict("code" => "publication_failed", "message" => "Publication operation failed locally. Review the journal and retry the same operation.") + _save(root, s, "operation_failed") +end + +function _finish!(root, s, deposit, client) + _remote_metadata(s, deposit, client) + _remote_files(s, deposit) + _local_integrity(root, s) + get(deposit, "submitted", false) === true && get(deposit, "state", "") == "done" || return false + doi = Zenodo.checked_doi(client, get(deposit, "doi", get(deposit["metadata"], "doi", nothing))) + doi == s["reserved_doi"] || throw(PublicationError(409, "doi_changed", "Published DOI differs from the reviewed DOI.")) + s["doi"] = doi + # Do not trust remote URLs (or render javascript: links). Construct known origins. + s["record_url"] = Zenodo.origin(client) * "/records/" * Zenodo.checked_id(get(deposit, "record_id", s["deposition_id"])) + receipt_path = joinpath(publication_path(root, s["id"]), "publication-receipt.json") + if isfile(receipt_path) + previous = read_json(receipt_path) + previous["doi"] == doi && previous["bundle_sha256"] == s["bundle_sha256"] || + throw(PublicationError(409, "receipt_conflict", "An immutable publication receipt already exists with different content.")) + s["published_at"] = previous["published_at"] + else + s["published_at"] = string(now(UTC)) * "Z" + # The sidecar is written BEFORE the terminal journal state. A crash here + # is recoverable by GET/reconciliation, without a second publish POST. + value = _public(merge(copy(s), Dict("state" => "published"))) + value["api_version"] = Zenodo.API_VERSION + value["events"] = vcat(s["events"], [Dict("at" => s["published_at"], "event" => "publication_verified", "state" => "published")]) + atomic_json(receipt_path, value) + end + s["state"] = "published" + s["last_error"] = nothing + _save(root, s, "publication_verified") + return true +end + +function _prepare_locked!(root, s, client) + _client_matches(s, client) + if s["state"] in ("ready", "published", "publishing", "publication_uncertain") + _local_integrity(root, s) + return _public(s) + end + if s["state"] in ("creating", "creation_uncertain") + throw(PublicationError(409, "creation_uncertain", "A draft creation may already have succeeded. Find its deposition ID on Zenodo and use recovery; creation will not be replayed.")) + end + try + if isnothing(s["deposition_id"]) + s["state"] = "creating" + _save(root, s, "creation_started") # write-ahead, before the non-idempotent POST + deposit = try + Zenodo.create_deposition(client, zenodo_metadata(s["metadata"], s["binding"], s["id"])) + catch e + s["state"] = e isa Zenodo.RemoteError && !e.ambiguous ? "preparing" : "creation_uncertain" + rethrow() + end + # If the response is malformed, creation remains uncertain, never retried. + s["deposition_id"] = Zenodo.checked_id(get(deposit, "id", nothing)) + s["state"] = "draft" + _save(root, s, "deposition_created") + end + deposit = Zenodo.get_deposition(client, s["deposition_id"]) + doi = _remote_metadata(s, deposit, client) + s["reserved_doi"] = doi + _save(root, s, "doi_reserved") + _make_archive!(root, s) + if get(deposit, "submitted", false) === true + _finish!(root, s, deposit, client) || throw(PublicationError(409, "zenodo_pending", "Zenodo is still processing this deposition. Refresh its status.")) + return _public(s) + end + files = get(deposit, "files", nothing) + files isa AbstractVector || throw(PublicationError(502, "invalid_files", "Zenodo returned an invalid file list.")) + # No extra files can hitch a ride into a publication. An interrupted PUT + # to our one filename is safe to repeat with the same frozen bytes. + all(f -> get(f, "filename", get(f, "name", get(f, "key", nothing))) == _archive_name(s["id"]), files) || + throw(PublicationError(409, "unexpected_remote_files", "The draft contains unexpected files. Review it on Zenodo before continuing.")) + if !(length(files) == 1 && _file_matches(only(files), s)) + _save(root, s, "upload_started") + uploaded = Zenodo.upload_file(client, deposit, _local_integrity(root, s), _archive_name(s["id"])) + _file_matches(uploaded, s) || throw(PublicationError(502, "upload_integrity_failed", "Zenodo's uploaded file checksum or size does not match the archive. Publishing is blocked.")) + end + verified = Zenodo.get_deposition(client, s["deposition_id"]) + _remote_metadata(s, verified, client) + _remote_files(s, verified) + get(verified, "submitted", false) === true && throw(PublicationError(409, "remote_state_changed", "The draft was submitted outside this operation. Refresh to reconcile it.")) + s["state"] = "ready" + s["last_error"] = nothing + _save(root, s, "draft_verified") + return _public(s) + catch e + s["state"] == "creating" && (s["state"] = "creation_uncertain") + _remember_error!(root, s, e) + rethrow() + end +end + +"""Freeze a local bundle, create a draft, reserve its DOI and verify its upload. Never publish.""" +function prepare!(root::String, bundle::String, input::AbstractDict, client::Zenodo.Client) + metadata = validate_metadata(input) + binding = snapshot(bundle) # all local validation BEFORE network side effects + isnothing(Sys.which("zip")) && throw(PublicationError(503, "zip_unavailable", "Install zip before preparing a Zenodo draft.")) + sum(filesize(joinpath(bundle, f)) for f in readdir(bundle)) <= MAX_ARCHIVE_BYTES || + throw(PublicationError(422, "bundle_too_large", "Bundle exceeds the supported 50 GB limit.")) + id = bytes2hex(sha256(canonical_json(Dict("environment" => client.environment, "binding" => binding)))) + directory = publication_path(root, id) + return with_publication_lock(directory) do + if isfile(_state_path(root, id)) + s = _read(root, id) + canonical_json(s["metadata"]) == canonical_json(metadata) || + throw(PublicationError(409, "metadata_frozen", "This exact config/result already has a publication with different metadata. Resume that record; do not mint a duplicate.")) + else + source = joinpath(directory, "source") + isdir(source) && rm(source; recursive=true) + cp(bundle, source) + # Verify the copied snapshot too: a caller must not race the freeze. + snapshot(source) == binding || throw(PublicationError(409, "bundle_changed", "The source bundle changed while it was being frozen.")) + timestamp = string(now(UTC)) * "Z" + s = Dict{String,Any}("schema_version" => SCHEMA_VERSION, "id" => id, "state" => "preparing", + "environment" => client.environment, "binding" => binding, "metadata" => metadata, + "deposition_id" => nothing, "reserved_doi" => nothing, "doi" => nothing, "record_url" => nothing, + "bundle_sha256" => nothing, "bundle_md5" => nothing, "bundle_size" => nothing, + "created_at" => timestamp, "updated_at" => timestamp, "published_at" => nothing, + "last_error" => nothing, "events" => Any[]) + _save(root, s, "bundle_snapshot_saved") + end + return _prepare_locked!(root, s, client) + end +end + +function resume!(root, id, client) + with_publication_lock(publication_path(root, id)) do + _prepare_locked!(root, _read(root, id), client) + end +end + +function recover_creation!(root, id, deposition_id, client) + with_publication_lock(publication_path(root, id)) do + s = _read(root, id) + _client_matches(s, client) + s["state"] in ("creating", "creation_uncertain") && isnothing(s["deposition_id"]) || + throw(PublicationError(409, "recovery_not_needed", "Only an uncertain draft creation can be attached to an existing deposition.")) + candidate = Zenodo.checked_id(deposition_id) + deposit = Zenodo.get_deposition(client, candidate) + # Validate the publication-specific metadata marker BEFORE persisting any ID. + proposed = merge(copy(s), Dict("deposition_id" => candidate)) + _remote_metadata(proposed, deposit, client) + isempty(get(deposit, "files", [])) && get(deposit, "submitted", false) === false || + throw(PublicationError(409, "unsafe_recovery", "Creation recovery requires the matching unsubmitted, empty draft.")) + s["deposition_id"] = candidate + s["state"] = "draft" + _save(root, s, "creation_recovered") + _prepare_locked!(root, s, client) + end +end + +function refresh!(root, id, client) + with_publication_lock(publication_path(root, id)) do + s = _read(root, id) + _client_matches(s, client) + s["state"] == "published" && return _public(s) + isnothing(s["deposition_id"]) && return _public(s) + try + deposit = Zenodo.get_deposition(client, s["deposition_id"]) + _remote_metadata(s, deposit, client) + if !isnothing(s["bundle_sha256"]) + _remote_files(s, deposit) + _finish!(root, s, deposit, client) + end + return _public(s) + catch e + _remember_error!(root, s, e) + rethrow() + end + end +end + +"""Publish once, only after confirmation of the exact environment, ID and archive SHA-256.""" +function publish!(root, id, client; confirmation, bundle_sha256, acknowledge_public=false) + with_publication_lock(publication_path(root, id)) do + s = _read(root, id) + _client_matches(s, client) + # Even a repeat of a published operation must name the reviewed artifact. + confirmation isa AbstractString && confirmation == confirmation_phrase(s) && + bundle_sha256 isa AbstractString && bundle_sha256 == s["bundle_sha256"] && acknowledge_public === true || + throw(PublicationError(422, "confirmation_required", "Publishing is irreversible and makes every file public. Confirm the exact environment, deposition ID and archive hash, and acknowledge the privacy/licensing review.")) + s["state"] == "published" && return _public(s) + s["state"] == "ready" || throw(PublicationError(409, "publication_not_ready", "Only a verified draft may be published. Pending or uncertain submissions must be reconciled, never replayed.")) + try + _local_integrity(root, s) + deposit = Zenodo.get_deposition(client, s["deposition_id"]) + _remote_metadata(s, deposit, client) + _remote_files(s, deposit) + if get(deposit, "submitted", false) === true + _finish!(root, s, deposit, client) + return _public(s) + end + s["state"] = "publishing" + s["last_error"] = nothing + s["confirmation"] = Dict("phrase" => confirmation, "bundle_sha256" => bundle_sha256, "acknowledge_public" => true) + _save(root, s, "publication_confirmed") # write-ahead BEFORE the irreversible POST + response = try + Zenodo.publish_deposition(client, s["deposition_id"]) + catch e + s["state"] = e isa Zenodo.RemoteError && !e.ambiguous ? "ready" : "publication_uncertain" + rethrow() + end + # 202 is acceptance, NOT proof of a published record. A subsequent GET + # (possibly a later refresh after restart) must report submitted/done. + Zenodo.checked_id(get(response, "id", nothing)) == s["deposition_id"] || + throw(PublicationError(502, "deposition_mismatch", "Publish response did not identify the expected deposition. Refresh to reconcile.")) + _save(root, s, "publication_accepted") + verified = Zenodo.get_deposition(client, s["deposition_id"]) + _finish!(root, s, verified, client) + return _public(s) + catch e + s["state"] == "publishing" && (s["state"] = "publication_uncertain") + _remember_error!(root, s, e) + rethrow() + end + end +end + +function receipt(root, id) + s = _read(root, id) + s["state"] == "published" || throw(PublicationError(409, "not_published", "There is no published receipt yet.")) + read_json(joinpath(publication_path(root, id), "publication-receipt.json")) +end + +function download_path(root, id, kind) + s = _read(root, id) + if kind == "bundle" + return _local_integrity(root, s), "application/zip", _archive_name(id) + elseif kind == "receipt" + receipt(root, id) # enforce published state + return joinpath(publication_path(root, id), "publication-receipt.json"), "application/json", "publication-" * id * ".json" + elseif kind == "citation" + s["state"] == "published" || throw(PublicationError(409, "not_published", "Cite only a published record, not a reserved DOI.")) + return joinpath(publication_path(root, id), "payload", "CITATION.cff"), "text/yaml", "citation-" * id * ".cff" + end + throw(PublicationError(404, "file_not_found", "Unknown publication download.")) +end + +end # module DOIPublications diff --git a/src/doi/Storage.jl b/src/doi/Storage.jl new file mode 100644 index 00000000..9b42bba3 --- /dev/null +++ b/src/doi/Storage.jl @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MPL-2.0 +# Publication state is separate from immutable scientific objects. Never store secrets here. +module DOIStorage + +using JSON3, SHA, OrderedCollections + +export PublicationError, atomic_write, atomic_json, read_json, canonical_json, + with_publication_lock, file_sha256, private_dir + +struct PublicationError <: Exception + status::Int + code::String + message::String +end +Base.showerror(io::IO, e::PublicationError) = print(io, e.message) + +canonical(x::AbstractDict) = OrderedDict(String(k) => canonical(x[k]) for k in sort!(collect(keys(x)); by=string)) +canonical(x::AbstractVector) = canonical.(x) +canonical(x) = x +canonical_json(x) = JSON3.write(canonical(x)) +file_sha256(path::AbstractString) = open(io -> bytes2hex(sha256(io)), path) +read_json(path::AbstractString) = JSON3.read(read(path, String), Dict{String,Any}) + +function private_dir(path::AbstractString) + islink(path) && throw(PublicationError(409, "unsafe_storage", "Publication storage must not be a symlink.")) + mkpath(path; mode=0o700) + chmod(path, 0o700) + return path +end + +# Supported deployment targets are native Unix and WSL2. Kernel locks are released +# on process death: unlike age-based lockfiles they cannot expire during a slow upload. +# Do not silently substitute a process-local mutex on unsupported platforms. +function with_publication_lock(f::Function, directory::AbstractString) + Sys.isunix() || throw(PublicationError(503, "unsupported_storage", "DOI publication requires a Unix filesystem with flock support (including WSL2).")) + private_dir(directory) + path = joinpath(directory, ".lock") + islink(path) && throw(PublicationError(409, "unsafe_storage", "Publication lock must not be a symlink.")) + open(path, "a+") do io + chmod(path, 0o600) + acquired = ccall(:flock, Cint, (Cint, Cint), fd(io), 2 | 4) == 0 # LOCK_EX | LOCK_NB + acquired || throw(PublicationError(409, "publication_busy", "Another publication operation is running. Refresh its status before retrying.")) + try + return f() + finally + ccall(:flock, Cint, (Cint, Cint), fd(io), 8) # LOCK_UN + end + end +end + +function atomic_write(path::AbstractString, content::AbstractString) + private_dir(dirname(path)) + islink(path) && throw(PublicationError(409, "unsafe_storage", "Publication files must not be symlinks.")) + tmp, io = mktemp(dirname(path); cleanup=false) + try + chmod(tmp, 0o600) + write(io, content) + flush(io) + if Sys.isunix() + rc = ccall(:fsync, Cint, (Cint,), fd(io)) + rc == 0 || error("Could not synchronise publication state") + end + close(io) + # rename, not rm + mv: readers must see either complete version, never a gap. + Base.Filesystem.rename(tmp, path) + if Sys.isunix() + dirfd = ccall(:open, Cint, (Cstring, Cint), dirname(path), 0) + dirfd >= 0 || error("Could not open publication state directory") + try + ccall(:fsync, Cint, (Cint,), dirfd) == 0 || error("Could not synchronise publication state directory") + finally + ccall(:close, Cint, (Cint,), dirfd) + end + end + finally + isopen(io) && close(io) + isfile(tmp) && rm(tmp) + end + return path +end +atomic_json(path::AbstractString, value) = atomic_write(path, canonical_json(value) * "\n") + +end # module DOIStorage diff --git a/src/doi/Web.jl b/src/doi/Web.jl new file mode 100644 index 00000000..44ea7f79 --- /dev/null +++ b/src/doi/Web.jl @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: MPL-2.0 +# Julia-authored progressive-enhancement UI. The tiny JS adapter does transport +# and DOM updates only; scientific, metadata and lifecycle decisions stay in Julia. +module DOIWeb + +using JSON3, Dates + +export render_page, html_escape +html_escape(text) = replace(string(text), '&' => "&", '<' => "<", '>' => ">", '"' => """, '\'' => "'") + +function render_page(study, csrf, environment, enabled; selected_config="") + bootstrap = replace(JSON3.write(Dict("study" => study, "csrf" => csrf, "environment" => environment, + "enabled" => enabled, "selected_config" => selected_config)), '<' => "\\u003c", '>' => "\\u003e", '&' => "\\u0026") + example = JSON3.write(Dict("method" => "nb_glm", "formula" => "~ group", "metadata_columns" => ["group"], + "normalization" => Dict("method" => "size_factors"), "created_by" => "Replace with your name")) + return """ + + + + Publish an analysis · MetaManifold + + + + +
+ ← Back to studies +

METAMANIFOLD · EVIDENCE & REPRODUCIBILITY

+

Make an analysis citable

Study: $(html_escape(study))

+

Create a private Zenodo draft, review the exact archive, then explicitly publish its DOI.

+

$(environment == "production" ? "PRODUCTION — real, permanent DOI publication" : "SANDBOX — test records, not production citations")

+
+

Publication is disabled or no server token is configured. An operator must configure Zenodo on the server. Never paste a token into this page.

+
+ + + +

2. Review & publish

+

Only a published record has a DOI badge. A reserved identifier or accepted request is not proof of publication. Refreshing never retries publication.

+
Loading saved publications…
+
+ +
+ +

DANGER — irreversible public publication

+

Publishing registers a permanent DOI and makes every archived file public. Removing a local config will not retract the Zenodo record.

+


+  
+ + +
+
+
+ + +""" +end + +end # module DOIWeb diff --git a/src/doi/Zenodo.jl b/src/doi/Zenodo.jl new file mode 100644 index 00000000..48b887d6 --- /dev/null +++ b/src/doi/Zenodo.jl @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: MPL-2.0 +# Zenodo's documented deposition-v1 API, not the separate InvenioRDM records API. +module Zenodo + +using HTTP, JSON3, Dates + +export Client, RemoteError, environment_client, origin, create_deposition, + get_deposition, upload_file, publish_deposition, checked_id, reserved_doi, + checked_doi, retry_after_seconds, API_VERSION + +const API_VERSION = "deposition-v1" +const ORIGINS = Dict("sandbox" => "https://sandbox.zenodo.org", "production" => "https://zenodo.org") + +struct Secret + value::String +end +Base.show(io::IO, ::Secret) = print(io, "[REDACTED]") + +# No remote response bodies or underlying HTTP exceptions escape this boundary. +# Both can contain Authorization headers, request URLs, or reflected credentials. +struct RemoteError <: Exception + status::Int + code::String + message::String + ambiguous::Bool + retry_after::Union{Int,Nothing} +end +Base.showerror(io::IO, e::RemoteError) = print(io, e.message) + +function _http(method, url, headers, body) + HTTP.request(method, url, headers, body; + redirect=false, retry=false, status_exception=false, logerrors=false, + connect_timeout=10, readtimeout=120) +end + +struct Client{T,S,C} + environment::String + token::Secret + transport::T + sleeper::S + clock::C + attempts::Int + retry_budget::Int +end + +function Client(token::AbstractString; environment::String="sandbox", transport=_http, + sleeper=sleep, clock=() -> now(UTC), attempts::Int=4, retry_budget::Int=60) + haskey(ORIGINS, environment) || throw(ArgumentError("Zenodo environment must be sandbox or production")) + isempty(strip(token)) && throw(ArgumentError("A server-side Zenodo token is required")) + any(isspace, token) && throw(ArgumentError("Invalid server-side Zenodo token")) + 1 <= attempts <= 5 || throw(ArgumentError("Zenodo attempts must be between 1 and 5")) + 0 <= retry_budget <= 120 || throw(ArgumentError("Zenodo retry budget must be between 0 and 120 seconds")) + Client(environment, Secret(String(token)), transport, sleeper, clock, attempts, retry_budget) +end +Base.show(io::IO, c::Client) = print(io, "Zenodo.Client(environment=", c.environment, ", token=[REDACTED])") +origin(c::Client) = ORIGINS[c.environment] + +function environment_client(env=ENV) + get(env, "METAMANIFOLD_ZENODO_ENABLED", "false") == "true" || + throw(ArgumentError("Zenodo publication is disabled by the server operator")) + environment = get(env, "METAMANIFOLD_ZENODO_ENVIRONMENT", "sandbox") + haskey(ORIGINS, environment) || throw(ArgumentError("Zenodo environment must be sandbox or production")) + key = environment == "sandbox" ? "ZENODO_SANDBOX_TOKEN" : "ZENODO_TOKEN" + Client(get(env, key, ""); environment) +end + +function checked_id(value) + # Bool is an Integer in Julia, but never a deposition identifier. + text = value isa AbstractString || (value isa Integer && !(value isa Bool)) ? string(value) : "" + occursin(r"^[1-9][0-9]{0,17}$", text) || + throw(RemoteError(502, "invalid_zenodo_response", "Zenodo returned an invalid deposition identifier.", false, nothing)) + return text +end + +function checked_doi(c::Client, value) + prefix = c.environment == "sandbox" ? "10.5072/zenodo." : "10.5281/zenodo." + value isa AbstractString && startswith(value, prefix) && occursin(r"^[1-9][0-9]*$", value[length(prefix)+1:end]) || + throw(RemoteError(502, "invalid_zenodo_response", "Zenodo returned a DOI for an unexpected service or environment.", false, nothing)) + return String(value) +end +reserved_doi(c::Client, deposit) = checked_doi(c, get(get(get(deposit, "metadata", Dict()), "prereserve_doi", Dict()), "doi", nothing)) + +function retry_after_seconds(value::AbstractString, clock::DateTime=now(UTC)) + seconds = tryparse(Int, strip(value)) + !isnothing(seconds) && return max(0, seconds) + occursin(r"^[0-9]+$", strip(value)) && return typemax(Int) + # HTTP-date (RFC 9110 IMF-fixdate); ignore malformed hints, not valid long waits. + endswith(value, " GMT") || return nothing + try + date = DateTime(value[1:end-4], dateformat"e, dd u yyyy HH:MM:SS") + return max(0, ceil(Int, Dates.value(date - clock) / 1000)) + catch + return nothing + end +end + +function _remote_error(status; ambiguous=false, retry_after=nothing) + code, message = if status == 401 + ("zenodo_unauthorized", "Zenodo rejected the server token. Ask the operator to check its environment and validity.") + elseif status == 403 + ("zenodo_forbidden", "Zenodo refused this operation. Check ownership and deposit:write / deposit:actions scopes.") + elseif status == 404 + ("zenodo_not_found", "The Zenodo draft was not found. Check the account and environment; no replacement was created.") + elseif status == 429 + ("zenodo_rate_limited", "Zenodo rate limit reached. Wait before retrying the same operation.") + elseif status in (400, 409, 415, 422) + ("zenodo_rejected", "Zenodo rejected the draft or its metadata. Review the draft on Zenodo; no DOI is claimed as published.") + elseif 300 <= status < 400 + ("zenodo_redirect_refused", "Zenodo redirected a credentialed request. Redirects are refused for token safety.") + else + ("zenodo_unavailable", "Zenodo did not return a usable response. Refresh or reconcile the existing publication; do not create a replacement.") + end + RemoteError(status, code, message, ambiguous, retry_after) +end + +# Factory bodies are reopened on EVERY attempt, including streamed uploads. HTTP.jl's +# automatic retries are disabled so an uncertain POST is never silently replayed. +function _request(c::Client, method::String, url::String; body_factory=() -> "", content_type="application/json", expected=(200,)) + # All endpoints are built here; only the bucket link is taken from a response, + # and it goes through a stricter origin/path validator below. + startswith(url, origin(c) * "/api/") || throw(ArgumentError("Refusing foreign Zenodo endpoint")) + headers = ["Authorization" => "Bearer " * c.token.value, + "Content-Type" => content_type, "Accept" => "application/json", + "User-Agent" => "MetaManifold-WebUI/doi-v1"] + safe = method in ("GET", "PUT") + waited = 0 + for attempt in 1:c.attempts + response = nothing + body = body_factory() + try + response = c.transport(method, url, headers, body) + catch + if !safe || attempt == c.attempts + throw(_remote_error(503; ambiguous=!safe)) + end + finally + body isa IO && close(body) + end + if !isnothing(response) + if response.status in expected + try + result = JSON3.read(String(response.body), Dict{String,Any}) + return result + catch + throw(RemoteError(502, "invalid_zenodo_response", "Zenodo returned malformed JSON; reconcile the existing operation before retrying.", !safe, nothing)) + end + end + hint = retry_after_seconds(HTTP.header(response, "Retry-After", ""), c.clock()) + # A definite 429 is safe to retry even for POST. 5xx/transport failures + # on create or publish are ambiguous and must be reconciled instead. + retryable = response.status == 429 || (safe && response.status in (500, 502, 503, 504)) + retryable && attempt < c.attempts || + throw(_remote_error(response.status; ambiguous=!safe && (response.status >= 500 || response.status < 400 || response.status == 408), retry_after=hint)) + delay = isnothing(hint) ? 2^(attempt - 1) : hint + else + delay = 2^(attempt - 1) + end + # Never truncate Retry-After and then retry too early. Return the hint to + # the caller when the server requests a wait beyond our bounded budget. + if delay > c.retry_budget - waited + isnothing(response) && throw(_remote_error(503)) + throw(_remote_error(response.status; retry_after=delay)) + end + c.sleeper(delay) + waited += delay + end + error("unreachable retry state") +end + +const DEPOSITIONS = "/api/deposit/depositions" +_endpoint(c, id) = origin(c) * DEPOSITIONS * "/" * checked_id(id) +create_deposition(c::Client, metadata) = _request(c, "POST", origin(c) * DEPOSITIONS; + body_factory=() -> JSON3.write(Dict("metadata" => metadata)), expected=(201,)) +get_deposition(c::Client, id) = _request(c, "GET", _endpoint(c, id)) +publish_deposition(c::Client, id) = _request(c, "POST", _endpoint(c, id) * "/actions/publish"; + body_factory=() -> "{}", expected=(200, 202)) + +function _bucket(c::Client, deposit) + url = get(get(deposit, "links", Dict()), "bucket", nothing) + url isa AbstractString || throw(RemoteError(502, "invalid_bucket", "Zenodo did not provide an upload bucket.", false, nothing)) + prefix = origin(c) * "/api/files/" + startswith(url, prefix) && occursin(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", url[length(prefix)+1:end]) || + throw(RemoteError(502, "unsafe_bucket", "Refusing an upload bucket outside the selected Zenodo origin or API path.", false, nothing)) + return String(url) +end + +function upload_file(c::Client, deposit, path::String, filename::String) + occursin(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,150}\.zip$", filename) || throw(ArgumentError("Invalid bundle filename")) + _request(c, "PUT", _bucket(c, deposit) * "/" * filename; + body_factory=() -> open(path, "r"), content_type="application/zip", expected=(200, 201)) +end + +end # module Zenodo diff --git a/src/doi/assets/publication.css b/src/doi/assets/publication.css new file mode 100644 index 00000000..d6294c19 --- /dev/null +++ b/src/doi/assets/publication.css @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MPL-2.0 */ +:root { color-scheme: light; font: 16px/1.6 system-ui, sans-serif; color: #243345; background: #f5f7fa; } +* { box-sizing: border-box; } +body { margin: 0; } +main { max-width: 980px; padding: 36px 24px 70px; margin: auto; } +h1 { font-size: clamp(1.8rem, 4vw, 2.6rem); line-height: 1.2; margin: 12px 0; } +h2 { font-size: 1.3rem; } h3 { margin: 0 0 8px; } +a { color: #145b80; text-underline-offset: 3px; } +header { padding: 20px 0; } +.eyebrow { font-size: .75rem; font-weight: 700; letter-spacing: .12em; color: #53677b; } +section, aside { background: white; padding: 24px; margin: 22px 0; border: 1px solid #dbe2eb; border-radius: 10px; } +.environment, .evidence { display: block; padding: 12px 16px; border-radius: 6px; font-weight: 650; } +.sandbox { background: #e9f2fa; color: #214a70; } +.production { background: #fff0e7; color: #8f3500; border: 1px solid #efb490; } +.evidence { background: #e7eeeb; border: 1px solid #abc3b7; } +label { display: block; margin: 15px 0; font-weight: 600; } +input:not([type=checkbox]), select, textarea { width: 100%; display: block; padding: 10px; border: 1px solid #9faebf; border-radius: 5px; background: white; color: inherit; font: inherit; margin-top: 6px; } +input[type=checkbox] { width: 18px; height: 18px; margin-right: 7px; vertical-align: middle; } +textarea { resize: vertical; } +fieldset { border: 0; padding: 0; min-width: 0; } +button { background: #1e5963; color: white; padding: 10px 16px; border: 1px solid transparent; border-radius: 5px; font: inherit; font-weight: 650; cursor: pointer; } +button.secondary { background: white; color: #34495e; border-color: #aebecb; } +button.danger { background: #a82e34; color: white; } +button:disabled { opacity: .55; cursor: not-allowed; } +:focus-visible { outline: 3px solid #118399; outline-offset: 3px; } +.columns { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; } +.actions, .section-heading { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; } +.section-heading { justify-content: space-between; } +.publication { border-top: 1px solid #dbe2eb; padding: 22px 0; } +.publication > a { display: inline-block; margin: 6px 14px 6px 0; } +.state { text-transform: none; font-weight: 600; color: #445f77; } +.warning, #error { background: #fff0ee; color: #9b2929; padding: 12px; border-left: 4px solid #b33333; } +#error { margin: 16px 0; } +#message:not(:empty) { background: #eaf4ed; padding: 12px; border-left: 4px solid #307e4f; } +pre, code { font-family: ui-monospace, monospace; overflow-wrap: anywhere; white-space: pre-wrap; } +pre { background: #f0f3f6; padding: 12px; font-size: .8rem; } +.doi-badge { padding: 6px 12px; border-radius: 4px; background: #e6f2ec; border: 1px solid #729d87; font-weight: 700; } +.ack { font-weight: 500; font-size: .92rem; } +details { margin-top: 24px; } summary { cursor: pointer; font-weight: 600; } +#config-json { font-family: ui-monospace, monospace; font-size: .85rem; } +dialog { width: min(92vw, 680px); max-height: 90vh; overflow: auto; border: 2px solid #b64040; border-radius: 10px; padding: 26px; } +dialog h2 { color: #a82e34; } dialog::backdrop { background: rgb(20 30 40 / 65%); } +[hidden] { display: none !important; } +@media (max-width: 600px) { main { padding: 20px 14px; } section, aside { padding: 16px; } .columns { grid-template-columns: 1fr; gap: 0; } .actions button { width: 100%; } } diff --git a/src/doi/assets/publication.js b/src/doi/assets/publication.js new file mode 100644 index 00000000..568f7306 --- /dev/null +++ b/src/doi/assets/publication.js @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: MPL-2.0 +// Progressive-enhancement adapter only. The Julia service owns validation, +// snapshot identities and every lifecycle transition. No tokens or localStorage. +const boot = JSON.parse(document.getElementById('doi-bootstrap').textContent) +const base = `/api/v1/studies/${encodeURIComponent(boot.study)}` +const byId = id => document.getElementById(id) +let configs = [] +let records = [] +let selected = null +let busy = false + +function element(tag, text, className) { + const node = document.createElement(tag) + if (text !== undefined) node.textContent = text + if (className) node.className = className + return node +} + +async function request(path, body) { + const response = await fetch(path, body === undefined ? { cache: 'no-store' } : { + method: 'POST', cache: 'no-store', + headers: { 'Content-Type': 'application/json', 'X-DOI-CSRF': boot.csrf }, + body: JSON.stringify(body), + }) + const data = await response.json().catch(() => ({ message: 'The server returned an unreadable response. Reload saved publications before retrying.' })) + if (!response.ok) { + const wait = response.headers.get('Retry-After') + throw new Error((data.message || 'Publication operation failed.') + (wait ? ` Retry after ${wait} seconds.` : '')) + } + return data +} + +function message(text = '') { byId('message').textContent = text } +function showError(error) { + byId('error').hidden = false + byId('error').textContent = error instanceof Error ? error.message : 'Publication operation failed.' +} +function clearError() { byId('error').hidden = true; byId('error').textContent = '' } + +function setBusy(value) { + busy = value + byId('prepare-fields').disabled = value || !boot.enabled + byId('reload').disabled = value + byId('config-form').querySelector('button').disabled = value + byId('evidence-mode').disabled = value + for (const button of byId('publications').querySelectorAll('button')) button.disabled = value || !boot.enabled + updateConfirmation() +} + +async function operation(fn) { + if (busy) return + clearError() + setBusy(true) + try { await fn() } catch (error) { showError(error); message('No success is assumed. Reload or reconcile the existing publication before retrying.') } + finally { + // Reload even after a lost response: the write-ahead journal may contain a + // newly created draft or an uncertain submission that must not be replayed. + try { await loadPublications() } catch (error) { showError(error) } + setBusy(false) + } +} + +function externalLink(text, url) { + const link = element('a', text) + // Defence in depth; the backend also constructs and validates every URL. + const parsed = new URL(url, location.origin) + if (parsed.protocol !== 'https:') return element('span', text) + link.href = parsed.href + link.target = '_blank' + link.rel = 'noopener noreferrer' + return link +} + +function action(text, run, className) { + const button = element('button', text, className) + button.type = 'button' + button.disabled = busy || !boot.enabled + button.addEventListener('click', () => { + if (!byId('evidence-mode').checked) { + showError(new Error('Enable Evidence Mode before taking a publication action.')) + byId('evidence-mode').focus() + return + } + run() + }) + return button +} + +function renderPublications() { + const host = byId('publications') + host.replaceChildren() + if (!records.length) { host.append(element('p', 'No saved publications. Preparing a draft never publishes it.')); return } + for (const record of records) { + const card = element('article', undefined, 'publication') + card.append(element('h3', record.metadata.title)) + card.append(element('p', `${record.environment.toUpperCase()} · ${record.state.replaceAll('_', ' ')} · ${record.binding.kind === 'configuration' ? 'Configuration only — no analysis results' : 'Selected analysis result'}`, 'state')) + card.append(element('p', `Config ${record.binding.config_id}${record.binding.result_id ? ` · Result ${record.binding.result_id}` : ''}`)) + if (record.binding.dangerous) card.append(element('p', 'DANGER: this configuration contains scientific overrides. Preserve and disclose the archived warning.', 'warning')) + if (record.bundle_sha256) { + card.append(element('pre', `Archive SHA-256\n${record.bundle_sha256}`)) + const download = element('a', 'Download exact archive for review') + download.href = `${base}/doi-publications/${record.id}/download/bundle` + card.append(download) + } + if (record.state === 'published') { + const badge = externalLink(`${record.test_record ? 'TEST DOI (sandbox)' : 'DOI'}: ${record.doi}`, record.doi_url) + badge.className = 'doi-badge' + card.append(badge, element('p', record.citation)) + for (const [kind, label] of [['receipt', 'Download provenance receipt'], ['citation', 'Download CITATION.cff']]) { + const link = element('a', label) + link.href = `${base}/doi-publications/${record.id}/download/${kind}` + card.append(link, document.createTextNode(' · ')) + } + } else { + if (record.reserved_doi) card.append(element('p', `Reserved identifier: ${record.reserved_doi} — not yet a published DOI.`)) + if (record.draft_url) card.append(externalLink('Review draft on Zenodo', record.draft_url)) + if (record.last_error) card.append(element('p', record.last_error.message, 'warning')) + const controls = element('div', undefined, 'actions') + if (['preparing', 'draft'].includes(record.state)) { + controls.append(action('Resume draft preparation', () => operation(async () => { + message('Resuming the same draft and frozen archive…') + await request(`${base}/doi-publications/${record.id}/resume`, {}) + message('Draft preparation completed. Review before publishing.') + }))) + } + if (['creating', 'creation_uncertain'].includes(record.state)) { + card.append(element('p', 'Creation outcome is uncertain. Find the existing draft with this publication marker on Zenodo. Never create a replacement automatically.')) + card.append(element('code', record.id)) + const label = element('label', 'Existing Zenodo deposition ID') + const input = element('input') + input.inputMode = 'numeric' + input.pattern = '[1-9][0-9]*' + label.append(input) + controls.append(label, action('Recover matching draft', () => operation(async () => { + await request(`${base}/doi-publications/${record.id}/recover`, { deposition_id: input.value.trim() }) + message('Recovered the matching draft; no replacement was created.') + }))) + } + if (record.state === 'ready') controls.append(action('Mint DOI…', () => openConfirmation(record), 'danger')) + if (record.deposition_id) controls.append(action('Refresh from Zenodo (never publishes)', () => operation(async () => { + message('Checking the existing Zenodo deposition…') + const current = await request(`${base}/doi-publications/${record.id}/refresh`, {}) + message(current.state === 'published' ? 'Publication verified. Your DOI is ready to cite.' : `Current state: ${current.state}. No publish request was sent.`) + }), 'secondary')) + if (['publishing', 'publication_uncertain'].includes(record.state)) card.append(element('p', 'Publication may still be processing. Refresh to reconcile it. If Zenodo still shows a draft after review, finish publication there manually, then refresh here; this application will not replay an uncertain publish request.')) + card.append(controls) + } + host.append(card) + } +} + +async function loadPublications() { + const data = await request(`${base}/doi-publications`) + records = data.publications + renderPublications() +} + +async function loadConfigs(preferred) { + const data = await request(`${base}/analysis-config`) + configs = data.configs + const select = byId('config-id') + select.replaceChildren(new Option(configs.length ? 'Choose an immutable saved configuration' : 'No saved configurations — use the JSON form below', '')) + for (const config of configs) select.add(new Option(`${config.method} · ${config.formula} · ${config.id}`, config.id)) + if (preferred && configs.some(c => c.id === preferred)) select.value = preferred + await chooseConfig() +} + +async function chooseConfig() { + const id = byId('config-id').value + const config = configs.find(c => c.id === id) + byId('config-summary').textContent = config ? `Config SHA-256: ${config.hash}${config.dangerous ? ' — DANGER: scientific overrides present' : ''}` : '' + byId('result-id').replaceChildren(new Option('Choose explicitly: configuration only or a completed result', '')) + if (!config) return + byId('result-id').add(new Option('Configuration only — no scientific result', 'config-only')) + const data = await request(`${base}/analysis-config/${encodeURIComponent(id)}/results`) + if (byId('config-id').value !== id) return // stale asynchronous selection + for (const result of data.results) { + const option = new Option(`${result.id} · ${result.publishable ? result.hash : 'mock/empty — not publishable'}`, result.id) + option.disabled = !result.publishable + byId('result-id').add(option) + } +} + +function openConfirmation(record) { + selected = record + byId('confirm-summary').textContent = `${record.environment.toUpperCase()} · ${record.metadata.title} · ${record.metadata.license} · ${record.binding.kind}` + byId('confirm-hash').textContent = `Deposition ${record.deposition_id}\nSHA-256 ${record.bundle_sha256}` + byId('expected-phrase').textContent = record.confirmation_phrase + byId('confirmation').value = '' + byId('public-ack').checked = false + updateConfirmation() + byId('publish-dialog').showModal() + byId('confirmation').focus() +} +function updateConfirmation() { + byId('confirm-publish').disabled = busy || !selected || !byId('public-ack').checked || byId('confirmation').value !== selected.confirmation_phrase +} + +byId('evidence-mode').addEventListener('change', () => { byId('advanced').hidden = !byId('evidence-mode').checked }) +byId('config-id').addEventListener('change', () => chooseConfig().catch(showError)) +byId('reload').addEventListener('click', () => operation(async () => { await loadConfigs(byId('config-id').value); message('Saved state reloaded. No Zenodo mutation was requested.') })) +byId('confirmation').addEventListener('input', updateConfirmation) +byId('public-ack').addEventListener('change', updateConfirmation) +byId('cancel-publish').addEventListener('click', () => { byId('publish-dialog').close(); selected = null }) +byId('publish-dialog').addEventListener('cancel', () => { selected = null }) + +byId('config-form').addEventListener('submit', event => { + event.preventDefault() + operation(async () => { + const data = await request(`${base}/analysis-config`, JSON.parse(byId('config-json').value)) + await loadConfigs(data.config.id) + message('Saved a new immutable configuration. No analysis was executed or DOI published.') + }) +}) +byId('prepare-form').addEventListener('submit', event => { + event.preventDefault() + operation(async () => { + const payload = byId('result-id').value + if (!payload || !byId('upload-ack').checked) throw new Error('Choose a payload and acknowledge the upload first.') + message('Preparing a private draft and verifying the frozen archive. Please wait…') + await request(`${base}/doi-publications`, { + config_id: byId('config-id').value, + result_id: payload === 'config-only' ? null : payload, + acknowledge_upload: true, + metadata: { + title: byId('title').value, description: byId('description').value, + creators: byId('creators').value.split('\n').map(name => name.trim()).filter(Boolean).map(name => ({ name })), + license: byId('license').value, version: byId('version').value, + publication_date: byId('publication-date').value, + github_release_url: byId('release-url').value || null, + github_project_url: byId('project-url').value || null, + }, + }) + message('Draft prepared — not published. Download and review its exact archive, then choose Mint DOI.') + }) +}) +byId('publish-form').addEventListener('submit', event => { + event.preventDefault() + if (!selected || byId('confirm-publish').disabled) return + const record = selected + const confirmation = byId('confirmation').value + byId('publish-dialog').close() + selected = null + operation(async () => { + message('Submitting the one confirmed publication request…') + const result = await request(`${base}/doi-publications/${record.id}/publish`, { + confirmation, bundle_sha256: record.bundle_sha256, acknowledge_public: true, + }) + message(result.state === 'published' ? 'Published and verified. Download the receipt and citation.' : 'Zenodo accepted the request but has not confirmed publication. Refresh the existing record; do not resubmit.') + }) +}) + +Promise.all([loadConfigs(boot.selected_config), loadPublications()]).catch(showError) diff --git a/src/server/routes/analysis_config.jl b/src/server/routes/analysis_config.jl index 7f0f1cf4..a34aca89 100644 --- a/src/server/routes/analysis_config.jl +++ b/src/server/routes/analysis_config.jl @@ -9,22 +9,16 @@ using MetaManifold: AnalysisConfig using MetaManifold.Epistemic using MetaManifold.CladeCumulus -# In-memory store for configs (in production, would be per-study persistent) -const _ANALYSIS_CONFIG_STORE = Dict{String,Dict{String,AnalysisConfig.AnalysisConfigStruct}}() # study -> id -> config -const _ANALYSIS_RESULT_STORE = Dict{String,Dict{String,AnalysisConfig.AnalysisResult}}() # study -> id -> result -const _ANALYSIS_CONFIG_LOCK = ReentrantLock() - -function _get_study_configs(study::String) - lock(_ANALYSIS_CONFIG_LOCK) do - get!(_ANALYSIS_CONFIG_STORE, study, Dict{String,AnalysisConfig.AnalysisConfigStruct}()) - end -end - -function _get_study_results(study::String) - lock(_ANALYSIS_CONFIG_LOCK) do - get!(_ANALYSIS_RESULT_STORE, study, Dict{String,AnalysisConfig.AnalysisResult}()) - end +# Durable per-study records; publication journals retain independent snapshots. +using MetaManifold: AnalysisStore, DOIBundles, DOIStorage +function _analysis_store_dir(study::String) + _valid_name(study) || throw(DOIStorage.PublicationError(400, "invalid_study", "Invalid study name.")) + project = joinpath(ServerState.projects_dir(), study) + islink(project) && throw(DOIStorage.PublicationError(403, "unsafe_storage", "Analysis storage must not be a symlink.")) + joinpath(project, ".analysis") end +_get_study_configs(study::String) = AnalysisStore.configs(_analysis_store_dir(study)) +_get_study_results(study::String) = AnalysisStore.results(_analysis_store_dir(study)) # List available metadata columns for a study (from first run's merged table or from study config) function _available_metadata_columns(study::String)::Vector{String} @@ -69,6 +63,8 @@ end normalization = AnalysisConfig.NormalizationConfig( method=String(norm_method), pseudocount=Float64(get(norm_body, "pseudocount", 0.5)), + epsilon=Float64(get(norm_body, "epsilon", 1e-6)), + zero_policy=String(get(norm_body, "zero_policy", "pseudocount")), ilr_basis=get(norm_body, "ilr_basis", nothing) isa Nothing ? nothing : String(get(norm_body, "ilr_basis", nothing)), multiplicative_replacement_delta=get(norm_body, "multiplicative_replacement_delta", nothing) isa Nothing ? nothing : Float64(get(norm_body, "multiplicative_replacement_delta", nothing)), # The declared scaling parameters travel with the request, are validated by the @@ -92,6 +88,9 @@ end advanced = AnalysisConfig.AdvancedOverrides( dispersion_method=String(get(adv_body, "dispersion_method", "parametric")), zero_handling=String(get(adv_body, "zero_handling", "pseudocount")), + zero_policy=String(get(adv_body, "zero_policy", get(adv_body, "zero_handling", "pseudocount"))), + pseudocount=Float64(get(adv_body, "pseudocount", 0.5)), + epsilon=Float64(get(adv_body, "epsilon", 1e-6)), min_prevalence=Float64(get(adv_body, "min_prevalence", 0.1)), min_abundance=Float64(get(adv_body, "min_abundance", 0.0)), max_features=get(adv_body, "max_features", nothing) isa Nothing ? nothing : Int(get(adv_body, "max_features", nothing)), @@ -115,7 +114,7 @@ end available_cols = _available_metadata_columns(study) errors = AnalysisConfig.validate_config(cfg, available_cols; strict=false) if !isempty(errors) - return json_error(400, "validation_failed", "AnalysisConfig validation failed", join(errors, "\n")) + return json_error(400, "validation_failed", "AnalysisConfig validation failed"; detail=join(errors, "\n")) end # Check DANGER @@ -126,10 +125,7 @@ end end # Store - lock(_ANALYSIS_CONFIG_LOCK) do - study_configs = _get_study_configs(study) - study_configs[cfg.id] = cfg - end + AnalysisStore.save_config!(_analysis_store_dir(study), cfg) # Return with danger banner if any resp = OrderedDict{String,Any}( @@ -146,7 +142,7 @@ end catch e if e isa ArgumentError - return json_error(400, "invalid_config", "Invalid AnalysisConfig: $(e.msg)", sprint(showerror, e)) + return json_error(400, "invalid_config", "Invalid AnalysisConfig: $(e.msg)"; detail=sprint(showerror, e)) else @error "Failed to create AnalysisConfig" exception=(e, catch_backtrace()) return json_error(500, "internal_error", "Failed to create AnalysisConfig: $(sprint(showerror, e))") @@ -195,11 +191,11 @@ end @delete "/api/v1/studies/{study}/analysis-config/{id}" function(req, study::String, id::String) study in _study_names() || return json_error(404, "study_not_found", "Study '$study' not found") - lock(_ANALYSIS_CONFIG_LOCK) do - study_configs = _get_study_configs(study) - haskey(study_configs, id) || return json_error(404, "config_not_found", "AnalysisConfig '$id' not found") - delete!(study_configs, id) - end + haskey(_get_study_configs(study), id) || return json_error(404, "config_not_found", "AnalysisConfig '$id' not found") + # Keep reviewed config identities addressable for both drafts and citations. + any(p -> p["binding"]["config_id"] == id, MetaManifold.DOIPublications.publications(_doi_store_dir(study))) && + return json_error(409, "config_has_publication", "This configuration has a DOI publication journal and cannot be deleted") + AnalysisStore.delete_config!(_analysis_store_dir(study), id) json(OrderedDict("deleted" => id)) end @@ -260,14 +256,12 @@ end "config_id" => cfg.id, "config_hash" => cfg.hash, "method" => AnalysisConfig.METHOD_TO_STRING[cfg.method], + "mock" => true, "note" => "Mock result — real implementation requires R packages DESeq2, compositions, etc. This is v1 scaffold with provenance chain intact.", ), ) - lock(_ANALYSIS_CONFIG_LOCK) do - study_results = _get_study_results(study) - study_results[result.id] = result - end + AnalysisStore.save_result!(_analysis_store_dir(study), result) json(OrderedDict( "result" => OrderedDict( @@ -297,48 +291,29 @@ end title = String(get(body, "title", "MetaManifold Analysis Bundle for $study")) license = String(get(body, "license", "CC-BY-4.0")) - # Find latest result for this config if any + # Explicit selection only: never bind a publication to a moving "latest" result. + result_id = get(body, "result_id", nothing) study_results = _get_study_results(study) - latest_result = nothing - for r in values(study_results) - if r.config_id == id - if isnothing(latest_result) || r.created_at > latest_result.created_at - latest_result = r - end - end - end - - mktempdir() do tmpdir - bundle_dir = joinpath(tmpdir, "bundle_$(cfg.id)") - bundle_path = AnalysisConfig.create_doi_bundle(cfg, latest_result, bundle_dir; authors, title, license) - - # Zip the bundle for download - zip_path = bundle_dir * ".zip" - try - run(`zip -r $zip_path $bundle_dir`) - catch - # Fallback: just return the directory path if zip fails - return json(OrderedDict( - "bundle_path" => bundle_path, - "config_id" => cfg.id, - "config_hash" => cfg.hash, - "doi_ready" => true, - "files" => readdir(bundle_path), - "datacite" => JSON3.read(read(joinpath(bundle_path, "datacite.json"), String)), - )) + !isnothing(result_id) && !haskey(study_results, result_id) && + return json_error(404, "result_not_found", "Selected analysis result was not found") + selected_result = isnothing(result_id) ? nothing : study_results[result_id] + try + mktempdir() do tmpdir + bundle_dir = joinpath(tmpdir, "bundle") + AnalysisConfig.create_doi_bundle(cfg, selected_result, bundle_dir; authors, title, license, + description=String(get(body, "description", "Analysis configuration export"))) + zip_path = joinpath(tmpdir, "bundle.zip") + DOIBundles.archive_bundle(bundle_dir, zip_path) + HTTP.Response(200, ["Content-Type" => "application/zip", "Cache-Control" => "no-store", + "Content-Disposition" => "attachment; filename=\"metamanifold-doi-bundle.zip\""]; body=read(zip_path)) end - - if isfile(zip_path) - data = read(zip_path) - HTTP.Response(200, ["Content-Type" => "application/zip", "Content-Disposition" => "attachment; filename=\"$(study)_$(id)_doi_bundle.zip\""], body=data) - else - json(OrderedDict( - "bundle_path" => bundle_path, - "config_id" => cfg.id, - "config_hash" => cfg.hash, - "doi_ready" => true, - )) + catch e + if e isa DOIStorage.PublicationError + return json_error(e.status, e.code, e.message) + elseif e isa ArgumentError + return json_error(422, "invalid_bundle", "The result does not belong to this exact configuration or the bundle is invalid") end + return json_error(500, "bundle_failed", "Could not create the DOI bundle; no temporary path is returned") end end diff --git a/src/server/routes/doi.jl b/src/server/routes/doi.jl new file mode 100644 index 00000000..fc284269 --- /dev/null +++ b/src/server/routes/doi.jl @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: MPL-2.0 +# Opt-in, local/single-user publication API. Never accepts tokens or remote API URLs. +using MetaManifold: Zenodo, DOIPublications, DOIWeb +using Random + +const _DOI_CSRF = bytes2hex(rand(RandomDevice(), UInt8, 32)) +const _DOI_CLIENT_FACTORY = Ref{Function}(() -> Zenodo.environment_client()) +_doi_client() = _DOI_CLIENT_FACTORY[]() + +function _doi_store_dir(study::String) + _valid_name(study) || throw(DOIStorage.PublicationError(400, "invalid_study", "Invalid study name.")) + project = joinpath(ServerState.projects_dir(), study) + islink(project) && throw(DOIStorage.PublicationError(403, "unsafe_storage", "DOI study storage must not be a symlink.")) + joinpath(project, ".doi") +end + +function _doi_capabilities() + environment = get(ENV, "METAMANIFOLD_ZENODO_ENVIRONMENT", "sandbox") + environment in ("sandbox", "production") || (environment = "sandbox") + enabled = try + client = _doi_client() + environment = client.environment + true + catch + false + end + Dict("enabled" => enabled, "environment" => environment, "api_version" => Zenodo.API_VERSION, "csrf" => _DOI_CSRF) +end + +# Browser mutations must be same-origin (or the one explicitly configured proxy +# origin) AND carry the per-process CSRF token fetched from the same-origin UI. +# This is CSRF protection, not user authentication. Do not expose this local API +# publicly without a separate authenticated reverse proxy. +function _doi_same_origin(req) + origin = HTTP.header(req, "Origin", "") + isempty(origin) && return true # non-browser clients still need JSON + CSRF + configured = get(ENV, "METAMANIFOLD_PUBLIC_ORIGIN", "") + !isempty(configured) && origin == configured && return true + uri = try HTTP.URIs.URI(origin) catch; return false end + uri.scheme in ("http", "https") && isempty(uri.userinfo) && isempty(uri.query) && isempty(uri.fragment) && isempty(uri.path) || return false + authority = uri.host * (isempty(uri.port) ? "" : ":" * uri.port) + lowercase(authority) == lowercase(HTTP.header(req, "Host", "")) +end + +function _doi_body(req; allowed=String[]) + HTTP.header(req, "X-DOI-CSRF", "") == _DOI_CSRF && _doi_same_origin(req) || + throw(DOIStorage.PublicationError(403, "publication_intent_required", "Reload the same-origin DOI page and retry. Its CSRF token or origin is missing or stale.")) + lowercase(strip(first(split(HTTP.header(req, "Content-Type", ""), ';')))) == "application/json" || + throw(DOIStorage.PublicationError(415, "json_required", "DOI operations require application/json.")) + length(req.body) <= 65_536 || throw(DOIStorage.PublicationError(413, "request_too_large", "DOI request exceeds 64 KiB.")) + body = try JSON3.read(String(copy(req.body)), Dict{String,Any}) catch + throw(DOIStorage.PublicationError(400, "invalid_json", "Expected a JSON object.")) + end + all(k -> k in allowed, keys(body)) || throw(DOIStorage.PublicationError(400, "unknown_field", "Unexpected DOI request field; tokens and API URLs must never be sent by the browser.")) + return body +end + +function _doi_json(value; status=200, headers=Pair{String,String}[]) + HTTP.Response(status, vcat(["Content-Type" => "application/json", "Cache-Control" => "no-store"], headers); body=JSON3.write(value)) +end + +function _doi_error(e) + if e isa DOIStorage.PublicationError + return _doi_json(Dict("error" => e.code, "message" => e.message); status=e.status) + elseif e isa Zenodo.RemoteError + status = e.status == 429 ? 429 : e.status in (400, 409, 415, 422) ? 422 : 502 + headers = isnothing(e.retry_after) ? Pair{String,String}[] : ["Retry-After" => string(e.retry_after)] + return _doi_json(Dict("error" => e.code, "message" => e.message, "outcome_uncertain" => e.ambiguous); status, headers) + elseif e isa ArgumentError + return _doi_json(Dict("error" => "publication_unavailable", "message" => "Check the server's Zenodo enable flag, environment and token, and the request values. No token is accepted through this API."); status=503) + end + # Do not log the exception or its backtrace: HTTP exceptions can carry headers. + @warn "DOI operation failed locally; inspect the publication journal (credentials omitted)" + _doi_json(Dict("error" => "publication_failed", "message" => "Publication failed locally. Reload saved publications and reconcile the existing operation before retrying."); status=500) +end + +function _doi_api(f::Function, req, study; mutation=false) + try + _valid_name(study) && study in _study_names() || throw(DOIStorage.PublicationError(404, "study_not_found", "Study not found.")) + if mutation + # Also excludes study rename/deletion while a network operation owns + # this journal. The per-record lock protects library/CLI callers too. + return DOIStorage.with_publication_lock(_doi_store_dir(study)) do + study in _study_names() || throw(DOIStorage.PublicationError(404, "study_not_found", "Study not found.")) + f() + end + end + f() + catch e + _doi_error(e) + end +end + +# Called by existing destructive study routes. Keeping a publication journal is +# part of the idempotency guarantee, not disposable UI cache. +function _doi_protect_study_mutation(f::Function, study) + try + DOIStorage.with_publication_lock(_doi_store_dir(study)) do + isempty(DOIPublications.publications(_doi_store_dir(study))) || + throw(DOIStorage.PublicationError(409, "study_has_publications", "This study has DOI publication journals. Keep its name and archive/back up the study instead of deleting its publication history.")) + f() + end + catch e + _doi_error(e) + end +end + +@get "/api/v1/doi/capabilities" function(req) + _doi_json(_doi_capabilities()) +end + +@get "/api/v1/doi/assets/{name}" function(req, name::String) + name in ("publication.js", "publication.css") || return json_error(404, "file_not_found", "Unknown DOI asset") + mime = endswith(name, ".js") ? "text/javascript" : "text/css" + path = joinpath(@__DIR__, "..", "..", "doi", "assets", name) + HTTP.Response(200, ["Content-Type" => mime, "X-Content-Type-Options" => "nosniff"]; body=read(path)) +end + +@get "/api/v1/studies/{study}/doi-ui" function(req, study::String) + _doi_api(req, study) do + capabilities = _doi_capabilities() + selected = get(HTTP.queryparams(req), "config", "") + html = DOIWeb.render_page(study, _DOI_CSRF, capabilities["environment"], capabilities["enabled"]; selected_config=selected) + HTTP.Response(200, ["Content-Type" => "text/html; charset=utf-8", "Cache-Control" => "no-store", + "X-Content-Type-Options" => "nosniff", "Referrer-Policy" => "no-referrer", + "Content-Security-Policy" => "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'self'"]; body=html) + end +end + +@get "/api/v1/studies/{study}/analysis-config/{id}/results" function(req, study::String, id::String) + _doi_api(req, study) do + haskey(_get_study_configs(study), id) || throw(DOIStorage.PublicationError(404, "config_not_found", "Saved configuration not found.")) + results = [Dict("id" => r.id, "hash" => r.hash, "created_at" => string(r.created_at), + "publishable" => !isempty(r.results) && get(r.provenance, "mock", false) !== true) + for r in values(_get_study_results(study)) if r.config_id == id] + _doi_json(Dict("results" => results)) + end +end + +@get "/api/v1/studies/{study}/doi-publications" function(req, study::String) + _doi_api(req, study) do + _doi_json(Dict("publications" => DOIPublications.publications(_doi_store_dir(study)))) + end +end + +@post "/api/v1/studies/{study}/doi-publications" function(req, study::String) + _doi_api(req, study; mutation=true) do + body = _doi_body(req; allowed=["config_id", "result_id", "metadata", "acknowledge_upload"]) + get(body, "acknowledge_upload", false) === true || throw(DOIStorage.PublicationError(422, "upload_consent_required", "Review privacy and consent to sending the selected bundle to Zenodo before preparing a draft.")) + haskey(body, "result_id") || throw(DOIStorage.PublicationError(422, "payload_selection_required", "Explicitly select result_id, or null for a configuration-only bundle. Latest-result selection is not supported.")) + config_id = get(body, "config_id", nothing) + config_id isa AbstractString || throw(DOIStorage.PublicationError(422, "config_required", "Select a saved configuration.")) + configs = _get_study_configs(study) + haskey(configs, config_id) || throw(DOIStorage.PublicationError(404, "config_not_found", "Saved configuration not found.")) + metadata = get(body, "metadata", nothing) + metadata isa AbstractDict || throw(DOIStorage.PublicationError(422, "metadata_required", "Explicit publication metadata is required.")) + metadata = DOIBundles.validate_metadata(metadata) + result_id = body["result_id"] + results = _get_study_results(study) + result = if isnothing(result_id) + nothing + elseif result_id isa AbstractString && haskey(results, result_id) + results[result_id] + else + throw(DOIStorage.PublicationError(404, "result_not_found", "Selected result not found.")) + end + client = _doi_client() + value = mktempdir() do tmp + bundle = joinpath(tmp, "bundle") + AnalysisConfig.create_doi_bundle(configs[config_id], result, bundle; + title=metadata["title"], authors=String[c["name"] for c in metadata["creators"]], + license=metadata["license"], description=metadata["description"]) + DOIPublications.prepare!(_doi_store_dir(study), bundle, metadata, client) + end + _doi_json(value) + end +end + +@get "/api/v1/studies/{study}/doi-publications/{id}" function(req, study::String, id::String) + _doi_api(req, study) do + _doi_json(DOIPublications.status(_doi_store_dir(study), id)) + end +end + +@post "/api/v1/studies/{study}/doi-publications/{id}/resume" function(req, study::String, id::String) + _doi_api(req, study; mutation=true) do + _doi_body(req) + _doi_json(DOIPublications.resume!(_doi_store_dir(study), id, _doi_client())) + end +end + +@post "/api/v1/studies/{study}/doi-publications/{id}/refresh" function(req, study::String, id::String) + _doi_api(req, study; mutation=true) do + _doi_body(req) + _doi_json(DOIPublications.refresh!(_doi_store_dir(study), id, _doi_client())) + end +end + +@post "/api/v1/studies/{study}/doi-publications/{id}/recover" function(req, study::String, id::String) + _doi_api(req, study; mutation=true) do + body = _doi_body(req; allowed=["deposition_id"]) + _doi_json(DOIPublications.recover_creation!(_doi_store_dir(study), id, get(body, "deposition_id", nothing), _doi_client())) + end +end + +@post "/api/v1/studies/{study}/doi-publications/{id}/publish" function(req, study::String, id::String) + _doi_api(req, study; mutation=true) do + body = _doi_body(req; allowed=["confirmation", "bundle_sha256", "acknowledge_public"]) + _doi_json(DOIPublications.publish!(_doi_store_dir(study), id, _doi_client(); + confirmation=get(body, "confirmation", nothing), bundle_sha256=get(body, "bundle_sha256", nothing), + acknowledge_public=get(body, "acknowledge_public", false))) + end +end + +@get "/api/v1/studies/{study}/doi-publications/{id}/download/{kind}" function(req, study::String, id::String, kind::String) + _doi_api(req, study) do + path, mime, name = DOIPublications.download_path(_doi_store_dir(study), id, kind) + HTTP.Response(200, ["Content-Type" => mime, "Cache-Control" => "no-store", + "Content-Disposition" => "attachment; filename=\"$name\"", "X-Content-Type-Options" => "nosniff"]; body=read(path)) + end +end diff --git a/src/server/routes/studies.jl b/src/server/routes/studies.jl index 6dd158ee..4b580720 100644 --- a/src/server/routes/studies.jl +++ b/src/server/routes/studies.jl @@ -225,9 +225,11 @@ end "Study '$new_name' already exists") _study_has_active_jobs(study) && return json_error(409, "jobs_active", "Study '$study' has active jobs - wait for them to finish before renaming") - _safe_move(joinpath(ServerState.data_dir(), study), joinpath(ServerState.data_dir(), new_name)) - _safe_move(joinpath(ServerState.projects_dir(), study), joinpath(ServerState.projects_dir(), new_name)) - json(_study_data(new_name)) + _doi_protect_study_mutation(study) do + _safe_move(joinpath(ServerState.data_dir(), study), joinpath(ServerState.data_dir(), new_name)) + _safe_move(joinpath(ServerState.projects_dir(), study), joinpath(ServerState.projects_dir(), new_name)) + json(_study_data(new_name)) + end end @delete "/api/v1/studies/{study}" function(req, study::String) @@ -235,9 +237,11 @@ end "Study '$study' not found") _study_has_active_jobs(study) && return json_error(409, "jobs_active", "Study '$study' has active jobs - wait for them to finish before deleting") - rm(joinpath(ServerState.data_dir(), study); recursive=true, force=true) - rm(joinpath(ServerState.projects_dir(), study); recursive=true, force=true) - json((; deleted=study)) + _doi_protect_study_mutation(study) do + rm(joinpath(ServerState.data_dir(), study); recursive=true, force=true) + rm(joinpath(ServerState.projects_dir(), study); recursive=true, force=true) + json((; deleted=study)) + end end ## Group management diff --git a/src/server/server.jl b/src/server/server.jl index 4c1622b9..ca6b6ec1 100644 --- a/src/server/server.jl +++ b/src/server/server.jl @@ -88,6 +88,7 @@ module Server include(joinpath(@__DIR__, "routes", "analysis.jl")) include(joinpath(@__DIR__, "routes", "composition.jl")) include(joinpath(@__DIR__, "routes", "analysis_config.jl")) + include(joinpath(@__DIR__, "routes", "doi.jl")) ## R-runtime busy middleware # The embedded R interpreter is shared between the pipeline and the analysis @@ -115,15 +116,17 @@ module Server # Same-origin request - no CORS headers needed return next(req) end - # Only allow localhost origins + # Same-origin proxy/preview deployments are permitted. Split remote + # deployments must name their public origin explicitly; this is not + # an authentication mechanism for exposing the local backend. origin_url = try HTTP.URIs.URI(origin) catch; nothing end - if isnothing(origin_url) || !(lowercase(origin_url.host) in ("localhost", "127.0.0.1", "::1")) + if !_doi_same_origin(req) && (isnothing(origin_url) || !(lowercase(origin_url.host) in ("localhost", "127.0.0.1", "::1"))) return HTTP.Response(403, "Forbidden: non-localhost origin") end cors_headers = [ "Access-Control-Allow-Origin" => origin, "Access-Control-Allow-Methods" => "GET, POST, PUT, PATCH, DELETE, OPTIONS", - "Access-Control-Allow-Headers" => "Content-Type", + "Access-Control-Allow-Headers" => "Content-Type, X-DOI-CSRF", ] # Handle preflight if req.method == "OPTIONS" @@ -155,6 +158,10 @@ module Server # rest may be {run}/... or {group}/{run}/... (group paths have an extra segment) m = match(r"^/files/([^/]+)/runs/(.+)$", uri) if !isnothing(m) + # Private analysis/publication journals are not generic run files. + # They are available only through the explicit DOI API allowlist. + segments = split(replace(HTTP.URIs.unescapeuri(m[2]), '\\' => '/'), '/') + any(segment -> startswith(segment, "."), segments) && return HTTP.Response(403, "Private application state") candidate = abspath(joinpath(ServerState.projects_dir(), HTTP.URIs.unescapeuri(m[1]), HTTP.URIs.unescapeuri(m[2]))) isfile(candidate) || return HTTP.Response(404, "File not found") full = realpath(candidate) diff --git a/test/doi/Project.toml b/test/doi/Project.toml new file mode 100644 index 00000000..4e5913b8 --- /dev/null +++ b/test/doi/Project.toml @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: MPL-2.0 +# Credential-free DOI contract lane. No R, biological databases, or live Zenodo. +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[compat] +HTTP = "=1.11.0" +JSON3 = "=1.14.3" +MD5 = "=0.2.3" +OrderedCollections = "=1.8.1" +julia = "1.12" diff --git a/test/doi/bootstrap.jl b/test/doi/bootstrap.jl new file mode 100644 index 00000000..7b62f6de --- /dev/null +++ b/test/doi/bootstrap.jl @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MPL-2.0 +# Load the actual publication implementation without importing the scientific/R stack. +module DOIIsolated +const SOURCE = joinpath(@__DIR__, "..", "..", "src", "doi") +include(joinpath(SOURCE, "Storage.jl")) +include(joinpath(SOURCE, "Zenodo.jl")) +include(joinpath(SOURCE, "Bundles.jl")) +include(joinpath(SOURCE, "Publications.jl")) +include(joinpath(SOURCE, "Web.jl")) +end diff --git a/test/doi/fixtures.jl b/test/doi/fixtures.jl new file mode 100644 index 00000000..594c0f83 --- /dev/null +++ b/test/doi/fixtures.jl @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: MPL-2.0 +# Explicit synthetic contract fixtures, never scientific estimates or live deposits. +const FAKE_TOKEN = "not-a-real-token-DO-NOT-LOG" +const CONFIG_ID = "12345678-1234-4234-8234-123456789012" +const RESULT_ID = "12345678-1234-4234-8234-123456789013" + +function metadata_fixture() + Dict{String,Any}("title" => "Reproducible analysis fixture", "description" => "Synthetic publication protocol fixture, not a scientific result.", + "creators" => [Dict("name" => "Example, Ada", "orcid" => "0000-0002-1825-0097")], + "license" => "CC-BY-4.0", "publication_date" => "2026-01-01", "version" => "1.0.0", + "github_release_url" => "https://github.com/example/research/releases/tag/v1.0.0", + "github_project_url" => "https://github.com/users/example/projects/1") +end + +function bundle_fixture(root; result=false, mock=false, dangerous=false) + dir = joinpath(root, "bundle") + mkpath(dir) + cfg = Dict("id" => CONFIG_ID, "hash" => repeat("a", 64), "dangerous" => dangerous, + "method" => "nb_glm", "created_at" => "2026-01-01T00:00:00", "created_by" => "Example, Ada") + write(joinpath(dir, "analysis_config.json"), JSON3.write(cfg)) + write(joinpath(dir, "analysis_config.ncl"), "{ method = \"nb_glm\" }\n") + write(joinpath(dir, "analysis_config_chora.deed"), "(repo-deed :schema-version \"1.0.0\" :dangerous #f)\n") + write(joinpath(dir, "provenance.json"), "{\"fixture\":true}") + write(joinpath(dir, "datacite.json"), "{}") + write(joinpath(dir, "README.md"), "# Synthetic protocol fixture\n") + write(joinpath(dir, "content_hash.txt"), cfg["hash"]) + dangerous && write(joinpath(dir, "DANGER_BANNER.txt"), "DANGER: synthetic unsafe configuration fixture") + if result + write(joinpath(dir, "analysis_result.json"), JSON3.write(Dict("id" => RESULT_ID, + "config_id" => CONFIG_ID, "config_hash" => cfg["hash"], "hash" => repeat("b", 64), + "method" => "nb_glm", "provenance" => Dict("mock" => mock), + "results" => Dict("synthetic_fixture" => Dict("status" => "fixture"))))) + end + B.write_checksums!(dir) + return dir +end + +mutable struct FakeZenodo + deposit::Dict{String,Any} + calls::Vector{Tuple{String,String}} + uploaded::Vector{UInt8} + faults::Vector{Function} + publish_done::Bool + environment::String +end +FakeZenodo(; environment="sandbox") = FakeZenodo(Dict{String,Any}(), Tuple{String,String}[], UInt8[], Function[], true, environment) + +function fake_response(value; status=200, headers=Pair{String,String}[]) + HTTP.Response(status, headers; body=JSON3.write(value)) +end + +function transport(fake::FakeZenodo) + return function(method, url, headers, body) + @test get(Dict(headers), "Authorization", "") == "Bearer " * FAKE_TOKEN + @test !occursin(FAKE_TOKEN, url) + @test !occursin("access_token", url) + @test startswith(url, Z.ORIGINS[fake.environment] * "/api/") + push!(fake.calls, (method, url)) + if !isempty(fake.faults) + fault = popfirst!(fake.faults) + response = fault(method, url, body) + !isnothing(response) && return response + end + if method == "POST" && endswith(url, "/deposit/depositions") + @test isempty(fake.deposit) # a duplicate create is a test failure + metadata = JSON3.read(String(body), Dict{String,Any})["metadata"] + prefix = fake.environment == "sandbox" ? "10.5072/zenodo." : "10.5281/zenodo." + metadata["prereserve_doi"] = Dict("doi" => prefix * "101", "recid" => 101) + fake.deposit = Dict{String,Any}("id" => 101, "state" => "unsubmitted", "submitted" => false, + "metadata" => metadata, "files" => Any[], "links" => Dict("bucket" => Z.ORIGINS[fake.environment] * "/api/files/11111111-1111-4111-8111-111111111111")) + return fake_response(fake.deposit; status=201) + elseif method == "GET" && endswith(url, "/101") + return fake_response(fake.deposit) + elseif method == "PUT" && occursin("/api/files/", url) + @test body isa IO # streaming upload, not an in-memory ZIP body + fake.uploaded = read(body) + file = Dict{String,Any}("name" => last(split(url, '/')), "checksum" => bytes2hex(md5(fake.uploaded)), "filesize" => string(length(fake.uploaded))) + fake.deposit["files"] = [file] + return fake_response(Dict("key" => file["name"], "checksum" => "md5:" * file["checksum"], "size" => length(fake.uploaded)); status=201) + elseif method == "POST" && endswith(url, "/101/actions/publish") + if fake.publish_done + mark_published!(fake) + end + return fake_response(fake.deposit; status=202) + end + error("Unexpected synthetic Zenodo request: $method $url") + end +end + +function mark_published!(fake) + fake.deposit["state"] = "done" + fake.deposit["submitted"] = true + fake.deposit["doi"] = fake.deposit["metadata"]["prereserve_doi"]["doi"] + fake.deposit["record_id"] = 101 +end +client(fake) = Z.Client(FAKE_TOKEN; environment=fake.environment, transport=transport(fake), sleeper=_ -> nothing) +count_calls(fake, method, suffix) = count(c -> c[1] == method && endswith(c[2], suffix), fake.calls) + +function prepared_fixture(f; result=false, dangerous=false, environment="sandbox") + mktempdir() do tmp + source = bundle_fixture(tmp; result, dangerous) + root = joinpath(tmp, "publications") + fake = FakeZenodo(; environment) + c = client(fake) + prepared = P.prepare!(root, source, metadata_fixture(), c) + f(tmp, root, source, fake, c, prepared) + end +end + +function publish_fixture(root, prepared, c) + P.publish!(root, prepared["id"], c; confirmation=prepared["confirmation_phrase"], + bundle_sha256=prepared["bundle_sha256"], acknowledge_public=true) +end + +function captured_error(f) + try f(); nothing catch e; e end +end diff --git a/test/doi/runtests.jl b/test/doi/runtests.jl new file mode 100644 index 00000000..f390cb22 --- /dev/null +++ b/test/doi/runtests.jl @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 +# julia --project=test/doi test/doi/runtests.jl +include("bootstrap.jl") +include("tests.jl") diff --git a/test/doi/tests.jl b/test/doi/tests.jl new file mode 100644 index 00000000..7e3ff41a --- /dev/null +++ b/test/doi/tests.jl @@ -0,0 +1,363 @@ +# SPDX-License-Identifier: MPL-2.0 +module DOIContractTests +using Test, HTTP, JSON3, Dates, SHA, MD5, Sockets +const Target = isdefined(Main, :MetaManifold) ? Main.MetaManifold : Main.DOIIsolated +const S = Target.DOIStorage +const B = Target.DOIBundles +const Z = Target.Zenodo +const P = Target.DOIPublications +const W = Target.DOIWeb +include("fixtures.jl") + +@testset "DOI metadata and bundle contracts" begin + input = metadata_fixture() + valid = B.validate_metadata(input) + @test valid["creators"][1]["orcid"] == "0000-0002-1825-0097" + @test valid["license"] == "CC-BY-4.0" + for field in ("title", "description", "creators", "license") + bad = deepcopy(input); delete!(bad, field) + @test_throws S.PublicationError B.validate_metadata(bad) + end + for (field, value) in (("title", " "), ("title", repeat("x", 251)), ("description", 42), + ("creators", []), ("creators", [Dict("name" => "Anonymous")]), + ("creators", [Dict("name" => "Example", "orcid" => "0000-0002-1825-0098")]), + ("license", "some-license"), ("publication_date", "2099-12-01"), + ("publication_date", "2026-02-30"), ("publication_date", "2026-1-1"), + ("github_release_url", "https://github.com/x/y/releases/latest"), + ("github_release_url", "https://github.com/x/y/releases/tag/../main"), + ("github_release_url", "https://github.com/x/y/releases/tag/v1?access_token=secret"), + ("github_release_url", "https://github.com/x/y/releases/tag/%2e%2e"), + ("github_project_url", "https://evil.example/users/x/projects/1"), + ("github_project_url", "javascript:alert(1)")) + bad = merge(deepcopy(input), Dict(field => value)) + @test_throws S.PublicationError B.validate_metadata(bad) + end + for secret_field in ("access_token", "token", "base_url", "doi", "access_right") + @test_throws S.PublicationError B.validate_metadata(merge(input, Dict(secret_field => "forbidden"))) + end + mktempdir() do tmp + bundle = bundle_fixture(tmp) + @test B.verify_checksums(bundle) + binding = B.snapshot(bundle) + @test binding["kind"] == "configuration" + @test isnothing(binding["result_id"]) + write(joinpath(bundle, "analysis_config.ncl"), "tampered") + @test_throws S.PublicationError B.snapshot(bundle) + end + for extra in ("secret.env", "unexpected.txt") + mktempdir() do tmp + bundle = bundle_fixture(tmp) + write(joinpath(bundle, extra), "must never upload") + @test_throws S.PublicationError B.snapshot(bundle) + end + end + mktempdir() do tmp + bundle = bundle_fixture(tmp) + rm(joinpath(bundle, "provenance.json")) + symlink(joinpath(bundle, "datacite.json"), joinpath(bundle, "provenance.json")) + @test_throws S.PublicationError B.snapshot(bundle) + end + mktempdir() do tmp + bundle = bundle_fixture(tmp; dangerous=true) + rm(joinpath(bundle, "DANGER_BANNER.txt")) + B.write_checksums!(bundle) + @test_throws S.PublicationError B.snapshot(bundle) + end + for modification in (:mock, :empty, :mismatch, :wrong_method, :not_run) + mktempdir() do tmp + bundle = bundle_fixture(tmp; result=true, mock=modification == :mock) + path = joinpath(bundle, "analysis_result.json") + result = S.read_json(path) + modification == :empty && (result["results"] = Dict()) + modification == :mismatch && (result["config_hash"] = repeat("c", 64)) + modification == :wrong_method && (result["method"] = "logistic") + modification == :not_run && (result["provenance"]["estimation"] = Dict("status" => "not_run")) + write(path, JSON3.write(result)); B.write_checksums!(bundle) + fake = FakeZenodo() + @test_throws S.PublicationError P.prepare!(joinpath(tmp, "state"), bundle, metadata_fixture(), client(fake)) + @test isempty(fake.calls) + end + end +end + +@testset "Zenodo client safety and bounded retries" begin + @test Z.origin(Z.Client(FAKE_TOKEN)) == "https://sandbox.zenodo.org" + @test_throws ArgumentError Z.Client(FAKE_TOKEN; environment="https://evil.example") + @test_throws ArgumentError Z.Client("header\ninjection") + @test_throws ArgumentError Z.environment_client(Dict("ZENODO_TOKEN" => FAKE_TOKEN)) + @test_throws ArgumentError Z.environment_client(Dict("METAMANIFOLD_ZENODO_ENABLED" => "true", "ZENODO_TOKEN" => FAKE_TOKEN)) + @test Z.environment_client(Dict("METAMANIFOLD_ZENODO_ENABLED" => "true", "ZENODO_SANDBOX_TOKEN" => FAKE_TOKEN)).environment == "sandbox" + @test !occursin(FAKE_TOKEN, sprint(show, Z.Client(FAKE_TOKEN))) + @test !occursin(FAKE_TOKEN, sprint(show, Z.Client(FAKE_TOKEN).token)) + @test Z.checked_id(123) == "123" + for id in (true, 0, -1, "1/../2", "1?access_token=x", 1.2, nothing) + @test_throws Z.RemoteError Z.checked_id(id) + end + @test_throws Z.RemoteError Z.checked_doi(Z.Client(FAKE_TOKEN), "10.5281/zenodo.123") + @test Z.retry_after_seconds("5") == 5 + @test Z.retry_after_seconds("nonsense") === nothing + @test Z.retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", DateTime(2015, 10, 21, 7, 27, 30)) == 30 + for status in (401, 403, 400, 404, 409, 415, 422, 302) + calls = Ref(0) + c = Z.Client(FAKE_TOKEN; transport=(args...) -> (calls[] += 1; fake_response(Dict("message" => FAKE_TOKEN); status)), sleeper=_ -> error("must not sleep")) + err = captured_error(() -> Z.get_deposition(c, 101)) + @test err isa Z.RemoteError + @test err.status == status + @test !occursin(FAKE_TOKEN, sprint(showerror, err)) + @test calls[] == 1 + end + for status in (429, 503) + calls = Ref(0); waits = Int[] + c = Z.Client(FAKE_TOKEN; transport=(args...) -> begin + calls[] += 1 + calls[] < 3 ? fake_response(Dict(); status, headers=["Retry-After" => "2"]) : fake_response(Dict("id" => 101)) + end, sleeper=x -> push!(waits, x)) + @test Z.get_deposition(c, 101)["id"] == 101 + @test calls[] == 3 + @test waits == [2, 2] + end + calls = Ref(0) + c = Z.Client(FAKE_TOKEN; transport=(args...) -> (calls[] += 1; fake_response(Dict(); status=429, headers=["Retry-After" => "3600"])), sleeper=_ -> error("must not retry early")) + err = captured_error(() -> Z.create_deposition(c, Dict())) + @test err.retry_after == 3600 + @test calls[] == 1 + calls[] = 0 + c = Z.Client(FAKE_TOKEN; transport=(args...) -> (calls[] += 1; fake_response(Dict(); status=429)), sleeper=_ -> nothing) + @test_throws Z.RemoteError Z.create_deposition(c, Dict()) + @test calls[] == 4 + for operation in (c -> Z.create_deposition(c, Dict()), c -> Z.publish_deposition(c, 101)) + for fault in (:server_error, :disconnect, :malformed) + calls[] = 0 + c = Z.Client(FAKE_TOKEN; transport=(args...) -> begin + calls[] += 1 + fault == :disconnect && error("Authorization: Bearer $FAKE_TOKEN") + fault == :malformed && return HTTP.Response(201; body="not JSON " * FAKE_TOKEN) + fake_response(Dict("message" => FAKE_TOKEN); status=503) + end, sleeper=_ -> error("must not retry ambiguous POST")) + err = captured_error(() -> operation(c)) + @test err isa Z.RemoteError + @test !occursin(FAKE_TOKEN, sprint(showerror, err)) + @test calls[] == 1 + # A status mismatch (201 on publish) is itself an unusable POST + # response; recovery must not automatically replay it. + end + end + mktempdir() do tmp + path = joinpath(tmp, "test.zip"); write(path, "test payload") + for bucket in ("http://sandbox.zenodo.org/api/files/11111111-1111-4111-8111-111111111111", + "https://evil.example/api/files/11111111-1111-4111-8111-111111111111", + "https://sandbox.zenodo.org.evil.example/api/files/11111111-1111-4111-8111-111111111111", + "https://zenodo.org/api/files/11111111-1111-4111-8111-111111111111", + "https://sandbox.zenodo.org/api/files/11111111-1111-4111-8111-111111111111?x=1", + "https://sandbox.zenodo.org/api/files/../deposit/depositions", + "https://user:pass@sandbox.zenodo.org/api/files/11111111-1111-4111-8111-111111111111") + c = Z.Client(FAKE_TOKEN; transport=(args...) -> error("must not send credentials")) + @test_throws Z.RemoteError Z.upload_file(c, Dict("links" => Dict("bucket" => bucket)), path, "test.zip") + end + bodies = String[] + c = Z.Client(FAKE_TOKEN; transport=(method, url, headers, body) -> begin + push!(bodies, String(read(body))) + length(bodies) == 1 && error("disconnect") + fake_response(Dict("ok" => true); status=201) + end, sleeper=_ -> nothing) + Z.upload_file(c, Dict("links" => Dict("bucket" => "https://sandbox.zenodo.org/api/files/11111111-1111-4111-8111-111111111111")), path, "test.zip") + @test bodies == ["test payload", "test payload"] + end +end + +@testset "Durable two-phase DOI publication" begin + for environment in ("sandbox", "production") + prepared_fixture(; result=true, dangerous=true, environment) do tmp, root, source, fake, c, prepared + @test prepared["state"] == "ready" + @test prepared["doi"] === nothing + @test prepared["doi_url"] === nothing + @test prepared["test_record"] == (environment == "sandbox") + @test startswith(prepared["reserved_doi"], environment == "sandbox" ? "10.5072/" : "10.5281/") + @test count_calls(fake, "POST", "/deposit/depositions") == 1 + @test count_calls(fake, "POST", "/actions/publish") == 0 + @test bytes2hex(sha256(fake.uploaded)) == prepared["bundle_sha256"] + @test length(fake.uploaded) == prepared["bundle_size"] + directory = P.publication_path(root, prepared["id"]) + payload = joinpath(directory, "payload") + @test B.verify_checksums(payload) + @test read(joinpath(payload, "analysis_config.json")) == read(joinpath(source, "analysis_config.json")) + @test read(joinpath(payload, "analysis_result.json")) == read(joinpath(source, "analysis_result.json")) + @test read(joinpath(payload, "provenance.json")) == read(joinpath(source, "provenance.json")) + @test isfile(joinpath(payload, "DANGER_BANNER.txt")) + @test S.read_json(joinpath(payload, "publication.json"))["state"] == "reserved" + @test S.read_json(joinpath(payload, "datacite.json"))["identifiers"][1]["identifier"] == prepared["reserved_doi"] + @test occursin(prepared["reserved_doi"], read(joinpath(payload, "CITATION.cff"), String)) + archive = P.download_path(root, prepared["id"], "bundle")[1] + entries = split(strip(read(`unzip -Z1 $archive`, String)), '\n') + @test Set(entries) == Set(readdir(payload)) + @test all(name -> !occursin('/', name), entries) # no leaked /tmp/ prefixes + @test success(`unzip -tq $archive`) + @test stat(joinpath(directory, "state.json")).mode & 0o777 == 0o600 + @test_throws S.PublicationError P.receipt(root, prepared["id"]) + @test_throws S.PublicationError P.download_path(root, prepared["id"], "citation") + before = length(fake.calls) + # New client object simulates reloading the service after restart; + # no process-local store participates in the idempotency decision. + repeated = P.prepare!(root, source, metadata_fixture(), client(fake)) + @test repeated["id"] == prepared["id"] + @test length(fake.calls) == before + @test P.status(root, prepared["id"])["bundle_sha256"] == prepared["bundle_sha256"] + @test length(P.publications(root)) == 1 + @test_throws S.PublicationError P.prepare!(root, source, merge(metadata_fixture(), Dict("title" => "Different")), c) + @test_throws S.PublicationError P.resume!(root, prepared["id"], Z.Client(FAKE_TOKEN; environment=environment == "sandbox" ? "production" : "sandbox")) + for kwargs in ((confirmation="wrong", bundle_sha256=prepared["bundle_sha256"], acknowledge_public=true), + (confirmation=prepared["confirmation_phrase"], bundle_sha256="wrong", acknowledge_public=true), + (confirmation=prepared["confirmation_phrase"], bundle_sha256=prepared["bundle_sha256"], acknowledge_public=false)) + @test_throws S.PublicationError P.publish!(root, prepared["id"], c; kwargs...) + end + @test count_calls(fake, "POST", "/actions/publish") == 0 + published = publish_fixture(root, prepared, c) + @test published["state"] == "published" + @test published["doi"] == prepared["reserved_doi"] + @test published["doi_url"] == "https://doi.org/" * published["doi"] + @test published["record_url"] == Z.origin(c) * "/records/101" + @test count_calls(fake, "POST", "/actions/publish") == 1 + @test P.receipt(root, prepared["id"])["bundle_sha256"] == prepared["bundle_sha256"] + @test P.receipt(root, prepared["id"])["binding"]["result_id"] == RESULT_ID + receipt_before = read(P.download_path(root, prepared["id"], "receipt")[1]) + before = length(fake.calls) + @test publish_fixture(root, prepared, client(fake))["doi"] == published["doi"] + @test length(fake.calls) == before + @test read(P.download_path(root, prepared["id"], "receipt")[1]) == receipt_before + @test S.file_sha256(archive) == prepared["bundle_sha256"] # no self-referential rewrite + for (dir, _, files) in walkdir(root), file in files + endswith(file, ".zip") && continue + @test !occursin(FAKE_TOKEN, read(joinpath(dir, file), String)) + end + end + end + prepared_fixture() do _, root, _, fake, c, prepared + @test prepared["binding"]["kind"] == "configuration" + @test occursin("Configuration only", fake.deposit["metadata"]["description"]) + @test P.refresh!(root, prepared["id"], c)["state"] == "ready" + @test count_calls(fake, "POST", "/actions/publish") == 0 + end +end + +@testset "Lost responses, recovery and integrity failures" begin + # Crash after creation is sent, before a deposition ID can be journalled. + mktempdir() do tmp + source = bundle_fixture(tmp); root = joinpath(tmp, "state") + fake = FakeZenodo(); normal = transport(fake) + faulty = Z.Client(FAKE_TOKEN; transport=(method, url, headers, body) -> begin + response = normal(method, url, headers, body) + method == "POST" && error("lost response with $FAKE_TOKEN") + response + end) + @test_throws Z.RemoteError P.prepare!(root, source, metadata_fixture(), faulty) + state = only(P.publications(root)) + @test state["state"] == "creation_uncertain" + @test state["deposition_id"] === nothing + @test_throws S.PublicationError P.prepare!(root, source, metadata_fixture(), client(fake)) + @test count_calls(fake, "POST", "/deposit/depositions") == 1 + notes = fake.deposit["metadata"]["notes"] + fake.deposit["metadata"]["notes"] = "Unrelated draft" + @test_throws S.PublicationError P.recover_creation!(root, state["id"], 101, client(fake)) + @test P.status(root, state["id"])["deposition_id"] === nothing + fake.deposit["metadata"]["notes"] = notes + recovered = P.recover_creation!(root, state["id"], 101, client(fake)) + @test recovered["state"] == "ready" + @test count_calls(fake, "POST", "/deposit/depositions") == 1 + end + # Definite rejection is retryable; an invalid response to a successful POST isn't. + for (status, body, expected) in ((401, "{}", "preparing"), (429, "{}", "preparing"), (201, "not-json", "creation_uncertain"), (201, "{}", "creation_uncertain")) + mktempdir() do tmp + source = bundle_fixture(tmp); root = joinpath(tmp, "state") + c = Z.Client(FAKE_TOKEN; transport=(args...) -> HTTP.Response(status; body), attempts=1) + @test !isnothing(captured_error(() -> P.prepare!(root, source, metadata_fixture(), c))) + @test only(P.publications(root))["state"] == expected + end + end + prepared_fixture() do _, root, _, fake, c, prepared + fake.publish_done = false + submitted = publish_fixture(root, prepared, c) + @test submitted["state"] == "publishing" + @test submitted["doi"] === nothing + @test_throws S.PublicationError publish_fixture(root, prepared, c) + @test count_calls(fake, "POST", "/actions/publish") == 1 + mark_published!(fake) + @test P.refresh!(root, prepared["id"], client(fake))["state"] == "published" + @test count_calls(fake, "POST", "/actions/publish") == 1 + end + prepared_fixture() do _, root, _, fake, _, prepared + normal = transport(fake) + faulty = Z.Client(FAKE_TOKEN; transport=(method, url, headers, body) -> begin + response = normal(method, url, headers, body) + method == "POST" && error("lost response with $FAKE_TOKEN") + response + end) + @test_throws Z.RemoteError publish_fixture(root, prepared, faulty) + @test P.status(root, prepared["id"])["state"] == "publication_uncertain" + @test_throws S.PublicationError publish_fixture(root, prepared, client(fake)) + @test P.refresh!(root, prepared["id"], client(fake))["state"] == "published" + @test count_calls(fake, "POST", "/actions/publish") == 1 + end + for modification in (:metadata, :extra_file, :checksum, :size, :local_bytes, :doi) + prepared_fixture() do _, root, _, fake, c, prepared + modification == :metadata && (fake.deposit["metadata"]["title"] = "Edited outside application") + modification == :extra_file && push!(fake.deposit["files"], Dict("name" => "secret.txt")) + modification == :checksum && (fake.deposit["files"][1]["checksum"] = repeat("0", 32)) + modification == :size && (fake.deposit["files"][1]["filesize"] = "1") + modification == :doi && (fake.deposit["metadata"]["prereserve_doi"]["doi"] = "10.5072/zenodo.999") + if modification == :local_bytes + open(P.download_path(root, prepared["id"], "bundle")[1], "a") do io; write(io, "changed"); end + end + @test_throws S.PublicationError publish_fixture(root, prepared, c) + @test count_calls(fake, "POST", "/actions/publish") == 0 + end + end + # PUT failure can be resumed with the original bytes and the same deposition. + mktempdir() do tmp + source = bundle_fixture(tmp); root = joinpath(tmp, "state") + fake = FakeZenodo(); normal = transport(fake) + faulty = Z.Client(FAKE_TOKEN; attempts=1, transport=(method, url, headers, body) -> begin + method == "PUT" && return fake_response(Dict(); status=503) + normal(method, url, headers, body) + end) + @test_throws Z.RemoteError P.prepare!(root, source, metadata_fixture(), faulty) + draft = only(P.publications(root)) + @test draft["state"] == "draft" + @test draft["bundle_sha256"] !== nothing + ready = P.resume!(root, draft["id"], client(fake)) + @test ready["bundle_sha256"] == draft["bundle_sha256"] + @test count_calls(fake, "POST", "/deposit/depositions") == 1 + end +end + +@testset "Private atomic storage and Julia UI" begin + mktempdir() do tmp + root = joinpath(tmp, "private") + S.atomic_json(joinpath(root, "state.json"), Dict("value" => 1)) + @test S.read_json(joinpath(root, "state.json"))["value"] == 1 + S.atomic_json(joinpath(root, "state.json"), Dict("value" => 2)) + @test S.read_json(joinpath(root, "state.json"))["value"] == 2 + @test readdir(root) == ["state.json"] + S.with_publication_lock(root) do + err = captured_error(() -> S.with_publication_lock(() -> nothing, root)) + @test err isa S.PublicationError + @test err.code == "publication_busy" + end + @test S.with_publication_lock(() -> true, root) + @test_throws S.PublicationError P.publication_path(root, "../../secrets") + @test_throws S.PublicationError P.publication_path(root, repeat("a", 63)) + symlink(joinpath(root, "state.json"), joinpath(root, "unsafe.json")) + @test_throws S.PublicationError S.atomic_json(joinpath(root, "unsafe.json"), Dict()) + end + html = W.render_page("", "csrf", "sandbox", false; selected_config="") + @test !occursin("