From 8f2ebf49242470ce68971f0f4193301d9b044869 Mon Sep 17 00:00:00 2001 From: Joe Corall <1189940+joecorall@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:12:59 +0000 Subject: [PATCH] [minor] Add generic Presentation storage and secure HTTP sources --- .dockerignore | 1 - Dockerfile | 12 +- Makefile | 7 +- cmd/triplet-benchmark-helper/main.go | 667 ++++++++++++++++++ cmd/triplet-benchmark-helper/main_test.go | 124 ++++ cmd/triplet-conformance-check/main.go | 98 +++ cmd/triplet-conformance-check/main_test.go | 25 + config.example.yaml | 19 +- deploy/compose/README.md | 18 +- deploy/compose/docker-compose.yaml | 47 +- ...7d7b76c96600ee21f44f2cb1741160c48e40.json} | 0 ...bb5661d505b93c01e636e245b66f34062db3.json} | 3 +- deploy/seccomp-triplet.json | 5 + docs/authorization.md | 26 + docs/caching.md | 15 +- docs/configuration.md | 63 +- docs/conformance.md | 21 +- docs/deploys.md | 9 +- go.mod | 18 +- go.sum | 46 +- internal/config/config.go | 24 +- internal/config/config_test.go | 86 ++- .../contracts/repository_contracts_test.go | 264 +++++++ .../iiif/presentation/v3/handler/handler.go | 373 ++++++---- .../presentation/v3/handler/handler_test.go | 613 ++++++++++------ internal/iiif/presentation/v3/store/file.go | 157 +++-- .../iiif/presentation/v3/store/file_test.go | 184 ++--- .../iiif/presentation/v3/store/mariadb.go | 131 ++-- .../presentation/v3/store/mariadb_test.go | 79 ++- .../v3/store/sql/queries/delete_resource.sql | 2 + .../sql/queries/insert_annotation_page.sql | 2 - .../v3/store/sql/queries/insert_resource.sql | 2 + .../sql/queries/select_annotation_page.sql | 1 - .../select_annotation_page_for_update.sql | 3 - .../v3/store/sql/queries/select_manifest.sql | 1 - .../v3/store/sql/queries/select_resource.sql | 3 + .../queries/select_resource_for_update.sql | 4 + .../sql/queries/update_annotation_page.sql | 3 - .../v3/store/sql/queries/update_resource.sql | 3 + .../v3/store/sql/schema/001_manifests.sql | 7 - .../v3/store/sql/schema/001_resources.sql | 7 + .../store/sql/schema/002_annotation_pages.sql | 8 - internal/iiif/presentation/v3/store/store.go | 52 +- .../iiif/presentation/v3/validate/validate.go | 282 ++------ .../presentation/v3/validate/validate_test.go | 86 ++- internal/server/server.go | 4 +- internal/storage/http.go | 77 +- internal/storage/http_test.go | 128 +++- scripts/benchmark-iiif.sh | 276 +------- scripts/conformance.sh | 192 ++--- scripts/ensure-iiif-validator.sh | 9 +- scripts/test-integration.sh | 2 + scripts/test.sh | 6 +- 53 files changed, 2948 insertions(+), 1347 deletions(-) create mode 100644 cmd/triplet-benchmark-helper/main.go create mode 100644 cmd/triplet-benchmark-helper/main_test.go create mode 100644 cmd/triplet-conformance-check/main.go create mode 100644 cmd/triplet-conformance-check/main_test.go rename deploy/compose/presentation/{item-1/canvas/canvas-1/annotations.json => resources/54/54b1ff4c782cca754695b23794617d7b76c96600ee21f44f2cb1741160c48e40.json} (100%) rename deploy/compose/presentation/{item-1/manifest.json => resources/cb/cbf1fbf4541d61625920ce657218bb5661d505b93c01e636e245b66f34062db3.json} (95%) create mode 100644 internal/contracts/repository_contracts_test.go create mode 100644 internal/iiif/presentation/v3/store/sql/queries/delete_resource.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/queries/insert_annotation_page.sql create mode 100644 internal/iiif/presentation/v3/store/sql/queries/insert_resource.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/queries/select_annotation_page.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/queries/select_annotation_page_for_update.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/queries/select_manifest.sql create mode 100644 internal/iiif/presentation/v3/store/sql/queries/select_resource.sql create mode 100644 internal/iiif/presentation/v3/store/sql/queries/select_resource_for_update.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/queries/update_annotation_page.sql create mode 100644 internal/iiif/presentation/v3/store/sql/queries/update_resource.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/schema/001_manifests.sql create mode 100644 internal/iiif/presentation/v3/store/sql/schema/001_resources.sql delete mode 100644 internal/iiif/presentation/v3/store/sql/schema/002_annotation_pages.sql diff --git a/.dockerignore b/.dockerignore index 7ec282b..fad060f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,7 +6,6 @@ results site deploy/compose/cache -deploy/compose/presentation deploy/compose/source-cache fixtures/benchmark diff --git a/Dockerfile b/Dockerfile index b51e24c..f4d35ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG \ # renovate: datasource=repology depName=debian_13/ca-certificates CA_CERTIFICATES_VERSION=20250419 \ # renovate: datasource=repology depName=debian_13/curl - CURL_VERSION=8.14.1-2+deb13u3 \ + CURL_VERSION=8.14.1-2+deb13u4 \ # renovate: datasource=repology depName=debian_13/libcgif-dev LIBCGIF_DEV_VERSION=0.5.0-1 \ # renovate: datasource=repology depName=debian_13/libexpat1-dev @@ -34,7 +34,7 @@ ARG \ # renovate: datasource=repology depName=debian_13/libspng-dev LIBSPNG_DEV_VERSION=0.7.4-2 \ # renovate: datasource=repology depName=debian_13/libtiff-dev - LIBTIFF_DEV_VERSION=4.7.0-3+deb13u2 \ + LIBTIFF_DEV_VERSION=4.7.0-3+deb13u3 \ # renovate: datasource=repology depName=debian_13/libwebp-dev LIBWEBP_DEV_VERSION=1.5.0-0.1 \ # renovate: datasource=repology depName=debian_13/meson @@ -170,13 +170,13 @@ ARG \ # renovate: datasource=repology depName=debian_13/liblcms2-2 LIBLCMS2_2_VERSION=2.16-2+deb13u2 \ # renovate: datasource=repology depName=debian_13/libopenjp2-7 - LIBOPENJP2_7_VERSION=2.5.3-2.1~deb13u1 \ + LIBOPENJP2_7_VERSION=2.5.3-2.1~deb13u2 \ # renovate: datasource=repology depName=debian_13/libpng16-16t64 LIBPNG16_16T64_VERSION=1.6.48-1+deb13u5 \ # renovate: datasource=repology depName=debian_13/libspng0 LIBSPNG0_VERSION=0.7.4-2 \ # renovate: datasource=repology depName=debian_13/libtiff6 - LIBTIFF6_VERSION=4.7.0-3+deb13u2 \ + LIBTIFF6_VERSION=4.7.0-3+deb13u3 \ # renovate: datasource=repology depName=debian_13/libwebp7 LIBWEBP7_VERSION=1.5.0-0.1 \ # renovate: datasource=repology depName=debian_13/libwebpdemux2 @@ -218,9 +218,11 @@ RUN rm -rf \ && useradd --system --gid triplet --uid 100 --home-dir /nonexistent --shell /usr/sbin/nologin triplet WORKDIR /var/lib/triplet -RUN mkdir -p /var/lib/triplet/cache /var/lib/triplet/testdata/images \ +RUN mkdir -p /var/lib/triplet/cache /var/lib/triplet/presentation \ + /var/lib/triplet/source-cache /var/lib/triplet/testdata/images \ && chown -R triplet:triplet /var/lib/triplet COPY --chown=triplet:triplet deploy/compose/images/ /var/lib/triplet/testdata/images/ +COPY --chown=triplet:triplet deploy/compose/presentation/ /var/lib/triplet/presentation/ COPY --from=build /out/triplet /usr/local/bin/triplet COPY --from=build /out/triplet-healthcheck /usr/local/bin/triplet-healthcheck diff --git a/Makefile b/Makefile index c8a8357..461247a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test test-integration test-race test-asan install-tools conformance benchmark-fixtures benchmark-iiif benchmark-iiif-pr lint generate fmt docker clean help docs-docker-build docs-build docs-serve docs-preview docs-clean +.PHONY: build test test-integration test-race test-asan install-tools conformance benchmark-fixtures benchmark-iiif benchmark-iiif-pr check-no-inline-python lint generate fmt docker clean help docs-docker-build docs-build docs-serve docs-preview docs-clean BIN ?= bin/triplet PKG ?= ./... @@ -42,7 +42,10 @@ benchmark-iiif: benchmark-iiif-pr: /bin/bash ./scripts/benchmark-iiif-pr.sh -lint: +check-no-inline-python: + go test ./internal/contracts -run TestShellScriptsDoNotInlinePython -count=1 + +lint: check-no-inline-python golangci-lint run $(PKG) fmt: diff --git a/cmd/triplet-benchmark-helper/main.go b/cmd/triplet-benchmark-helper/main.go new file mode 100644 index 0000000..ad9081b --- /dev/null +++ b/cmd/triplet-benchmark-helper/main.go @@ -0,0 +1,667 @@ +// Command triplet-benchmark-helper provides small, deterministic helpers for +// the IIIF benchmark shell driver. Keeping these operations in a compiled Go +// command avoids embedding another programming language in shell automation. +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/csv" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode" +) + +const usage = `usage: triplet-benchmark-helper [arguments] + +commands: + urlencode VALUE + hash VALUE [VALUE...] + epoch + update-run RUN_JSON STARTED_AT FINISHED_AT START_EPOCH FINISH_EPOCH + matrix-summary OUT_ROOT RUN_ID INDEX + append-matrix-reports OUT_ROOT RUN_ID INDEX` + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(args []string, stdout io.Writer) error { + if len(args) == 0 { + return errors.New(usage) + } + + switch args[0] { + case "urlencode": + if len(args) != 2 { + return errors.New("usage: triplet-benchmark-helper urlencode VALUE") + } + _, err := fmt.Fprintln(stdout, percentEncode(args[1])) + return err + case "hash": + if len(args) < 2 { + return errors.New("usage: triplet-benchmark-helper hash VALUE [VALUE...]") + } + _, err := fmt.Fprintln(stdout, hashValues(args[1:])) + return err + case "epoch": + if len(args) != 1 { + return errors.New("usage: triplet-benchmark-helper epoch") + } + _, err := fmt.Fprintf(stdout, "%.6f\n", float64(time.Now().UnixNano())/1e9) + return err + case "update-run": + if len(args) != 6 { + return errors.New("usage: triplet-benchmark-helper update-run RUN_JSON STARTED_AT FINISHED_AT START_EPOCH FINISH_EPOCH") + } + return updateRun(args[1], args[2], args[3], args[4], args[5]) + case "matrix-summary": + if len(args) != 4 { + return errors.New("usage: triplet-benchmark-helper matrix-summary OUT_ROOT RUN_ID INDEX") + } + return writeMatrixSummary(args[1], args[2], args[3]) + case "append-matrix-reports": + if len(args) != 4 { + return errors.New("usage: triplet-benchmark-helper append-matrix-reports OUT_ROOT RUN_ID INDEX") + } + return appendMatrixReports(args[1], args[2], args[3]) + default: + return fmt.Errorf("unknown command %q\n\n%s", args[0], usage) + } +} + +func percentEncode(value string) string { + const hexadecimal = "0123456789ABCDEF" + var encoded strings.Builder + encoded.Grow(len(value)) + for _, b := range []byte(value) { + if b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b >= '0' && b <= '9' || strings.ContainsRune("-._~", rune(b)) { + _ = encoded.WriteByte(b) + continue + } + _ = encoded.WriteByte('%') + _ = encoded.WriteByte(hexadecimal[b>>4]) + _ = encoded.WriteByte(hexadecimal[b&0x0f]) + } + return encoded.String() +} + +func hashValues(values []string) string { + hash := sha256.New() + for index, value := range values { + if index > 0 { + _, _ = hash.Write([]byte{0}) + } + _, _ = hash.Write([]byte(value)) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +func updateRun(path, startedAt, finishedAt, startedEpoch, finishedEpoch string) error { + started, err := strconv.ParseFloat(startedEpoch, 64) + if err != nil { + return fmt.Errorf("parse measured start epoch: %w", err) + } + finished, err := strconv.ParseFloat(finishedEpoch, 64) + if err != nil { + return fmt.Errorf("parse measured finish epoch: %w", err) + } + + run, err := readRun(path) + if err != nil { + return err + } + run["measured_started_at"] = startedAt + run["measured_finished_at"] = finishedAt + run["measured_duration_seconds"] = math.Round((finished-started)*1e6) / 1e6 + + var output bytes.Buffer + encoder := json.NewEncoder(&output) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(run); err != nil { + return fmt.Errorf("encode %s: %w", path, err) + } + return atomicWrite(path, output.Bytes()) +} + +type requestStats struct { + total int + ok int + times []float64 + sizes []float64 +} + +type matrixRow struct { + mode string + concurrency string + server string + cells []string +} + +func writeMatrixSummary(outRoot, runID, indexPath string) error { + runFiles, err := matrixRunFiles(outRoot, runID) + if err != nil { + return err + } + + var summaryRows []matrixRow + var overallRows []matrixRow + tripletImages := make(map[string]struct{}) + for _, runPath := range runFiles { + runDir := filepath.Dir(runPath) + requestsPath := filepath.Join(runDir, "requests.csv") + if _, err := os.Stat(requestsPath); errors.Is(err, os.ErrNotExist) { + continue + } else if err != nil { + return fmt.Errorf("stat %s: %w", requestsPath, err) + } + + run, err := readRun(runPath) + if err != nil { + return err + } + tripletImages[fieldString(run, "triplet_image", "-")] = struct{}{} + + byServer, err := readRequestStats(requestsPath) + if err != nil { + return err + } + resources, err := readResources(filepath.Join(runDir, "resource-summary.csv")) + if err != nil { + return err + } + + mode := fieldString(run, "mode", "-") + concurrency := fieldString(run, "concurrency", "-") + duration := fieldFloat(run, "measured_duration_seconds") + triplet := byServer["triplet"] + + var cpuPerRequest *float64 + if triplet != nil && triplet.ok > 0 { + if meanCPU, err := strconv.ParseFloat(resources["triplet"]["mean_cpu_percent"], 64); err == nil { + value := meanCPU / 100 * duration / float64(triplet.ok) + cpuPerRequest = &value + } + } + var maxMemory *float64 + if value, err := strconv.ParseFloat(resources["triplet"]["max_mem_mib"], 64); err == nil { + maxMemory = &value + } + + if triplet == nil { + triplet = &requestStats{} + } + summaryRows = append(summaryRows, matrixRow{ + mode: mode, + concurrency: concurrency, + cells: []string{ + mode, + concurrency, + formatRate(triplet.ok, triplet.total), + formatSeconds(duration), + formatRatePerSecond(triplet.ok, duration), + formatMilliseconds(percentile(triplet.times, 0.95), 1), + formatMilliseconds(percentile(triplet.times, 0.99), 1), + formatOptionalMilliseconds(cpuPerRequest, 2), + formatOptionalFloat(maxMemory, 1), + }, + }) + + servers := make([]string, 0, len(byServer)) + for server := range byServer { + servers = append(servers, server) + } + sort.Strings(servers) + for _, server := range servers { + stats := byServer[server] + overallRows = append(overallRows, matrixRow{ + mode: mode, + concurrency: concurrency, + server: server, + cells: []string{ + mode, + concurrency, + server, + formatRate(stats.ok, stats.total), + formatMilliseconds(median(stats.times), 1), + formatMilliseconds(mean(stats.times), 1), + formatSize(mean(stats.sizes)), + fmt.Sprintf("[report](../%s/report.md)", filepath.Base(runDir)), + }, + }) + } + } + + if len(summaryRows) == 0 { + return nil + } + + sort.Slice(summaryRows, func(i, j int) bool { return matrixRowLess(summaryRows[i], summaryRows[j]) }) + sort.Slice(overallRows, func(i, j int) bool { + if matrixRowLess(overallRows[i], overallRows[j]) { + return true + } + if matrixRowLess(overallRows[j], overallRows[i]) { + return false + } + return overallRows[i].server < overallRows[j].server + }) + + original, err := os.ReadFile(indexPath) + if err != nil { + return fmt.Errorf("read %s: %w", indexPath, err) + } + title, remainder, found := strings.Cut(string(original), "\n\n") + if !found { + remainder = "" + } + + images := make([]string, 0, len(tripletImages)) + for image := range tripletImages { + images = append(images, image) + } + sort.Strings(images) + + lines := []string{ + title, + "", + "## Summary", + "", + fmt.Sprintf("Triplet image: `%s`", strings.Join(images, ", ")), + "", + "| Mode | Concurrency | Triplet OK | Duration s | Req/s | p95 ms | p99 ms | CPU ms/req | Max MiB |", + "| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: |", + } + for _, row := range summaryRows { + lines = append(lines, "| "+strings.Join(row.cells, " | ")+" |") + } + lines = append(lines, "", "Status reflects Triplet request success. Performance metrics are informational.", "") + if strings.TrimSpace(remainder) != "" { + lines = append(lines, strings.TrimRightFunc(remainder, unicode.IsSpace), "") + } + lines = append(lines, + "## Overall Summary", + "", + "| Mode | Concurrency | Server | Success | Median ms | Mean ms | Mean bytes | Report |", + "| --- | ---: | --- | --- | ---: | ---: | ---: | --- |", + ) + for _, row := range overallRows { + lines = append(lines, "| "+strings.Join(row.cells, " | ")+" |") + } + + output := strings.TrimRightFunc(strings.Join(lines, "\n"), unicode.IsSpace) + "\n" + return atomicWrite(indexPath, []byte(output)) +} + +func appendMatrixReports(outRoot, runID, indexPath string) error { + runFiles, err := matrixRunFiles(outRoot, runID) + if err != nil { + return err + } + + var appended strings.Builder + _, _ = appended.WriteString("\n## Run Reports\n\n") + for _, runPath := range runFiles { + runDir := filepath.Dir(runPath) + reportPath := filepath.Join(runDir, "report.md") + report, err := os.ReadFile(reportPath) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return fmt.Errorf("read %s: %w", reportPath, err) + } + run, err := readRun(runPath) + if err != nil { + return err + } + _, _ = fmt.Fprintf(&appended, "### %s\n\n", filepath.Base(runDir)) + _, _ = fmt.Fprintf(&appended, "- Mode: `%s`\n", fieldString(run, "mode", "-")) + _, _ = fmt.Fprintf(&appended, "- Concurrency: `%s`\n", fieldString(run, "concurrency", "-")) + _, _ = fmt.Fprintf(&appended, "- Directory: `%s`\n\n", runDir) + _, _ = appended.WriteString(demoteHeadings(string(report))) + _ = appended.WriteByte('\n') + } + + file, err := os.OpenFile(indexPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return fmt.Errorf("open %s: %w", indexPath, err) + } + if _, err := io.WriteString(file, appended.String()); err != nil { + _ = file.Close() + return fmt.Errorf("append %s: %w", indexPath, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close %s: %w", indexPath, err) + } + return nil +} + +func demoteHeadings(markdown string) string { + markdown = strings.ReplaceAll(markdown, "\r\n", "\n") + markdown = strings.TrimRightFunc(markdown, unicode.IsSpace) + if markdown == "" { + return "\n" + } + lines := strings.Split(markdown, "\n") + for index, line := range lines { + if strings.HasPrefix(line, "#") { + lines[index] = "#" + line + } + } + return strings.Join(lines, "\n") + "\n" +} + +func matrixRunFiles(outRoot, runID string) ([]string, error) { + entries, err := os.ReadDir(outRoot) + if err != nil { + return nil, fmt.Errorf("read benchmark output root %s: %w", outRoot, err) + } + paths := make([]string, 0) + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), runID+"-") { + continue + } + path := filepath.Join(outRoot, entry.Name(), "run.json") + if _, err := os.Stat(path); err == nil { + paths = append(paths, path) + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("stat %s: %w", path, err) + } + } + sort.Strings(paths) + return paths, nil +} + +func readRun(path string) (map[string]any, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open %s: %w", path, err) + } + defer file.Close() + decoder := json.NewDecoder(file) + decoder.UseNumber() + var run map[string]any + if err := decoder.Decode(&run); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + return run, nil +} + +func readRequestStats(path string) (map[string]*requestStats, error) { + rows, err := readCSV(path) + if err != nil { + return nil, err + } + byServer := make(map[string]*requestStats) + for _, row := range rows { + server := row["server"] + stats := byServer[server] + if stats == nil { + stats = &requestStats{} + byServer[server] = stats + } + stats.total++ + if row["curl_exit"] != "0" || !strings.HasPrefix(row["http_code"], "2") { + continue + } + total, err := strconv.ParseFloat(row["time_total"], 64) + if err != nil { + return nil, fmt.Errorf("parse time_total in %s: %w", path, err) + } + size, err := strconv.ParseFloat(row["size_download"], 64) + if err != nil { + return nil, fmt.Errorf("parse size_download in %s: %w", path, err) + } + stats.ok++ + stats.times = append(stats.times, total) + stats.sizes = append(stats.sizes, math.Trunc(size)) + } + return byServer, nil +} + +func readResources(path string) (map[string]map[string]string, error) { + rows, err := readCSV(path) + if errors.Is(err, os.ErrNotExist) { + return make(map[string]map[string]string), nil + } + if err != nil { + return nil, err + } + resources := make(map[string]map[string]string, len(rows)) + for _, row := range rows { + resources[row["server"]] = row + } + return resources, nil +} + +func readCSV(path string) ([]map[string]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + reader := csv.NewReader(file) + headers, err := reader.Read() + if err != nil { + return nil, fmt.Errorf("read CSV header %s: %w", path, err) + } + var rows []map[string]string + for { + values, err := reader.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("read CSV row %s: %w", path, err) + } + row := make(map[string]string, len(headers)) + for index, header := range headers { + row[header] = values[index] + } + rows = append(rows, row) + } + return rows, nil +} + +func fieldString(run map[string]any, key, fallback string) string { + value, ok := run[key] + if !ok || value == nil { + return fallback + } + switch typed := value.(type) { + case string: + return typed + case json.Number: + return typed.String() + default: + return fmt.Sprint(typed) + } +} + +func fieldFloat(run map[string]any, key string) float64 { + value, ok := run[key] + if !ok || value == nil { + return 0 + } + switch typed := value.(type) { + case json.Number: + parsed, _ := typed.Float64() + return parsed + case float64: + return typed + case string: + parsed, _ := strconv.ParseFloat(typed, 64) + return parsed + default: + return 0 + } +} + +func matrixRowLess(left, right matrixRow) bool { + leftMode := modeOrder(left.mode) + rightMode := modeOrder(right.mode) + if leftMode != rightMode { + return leftMode < rightMode + } + leftConcurrency, leftErr := strconv.Atoi(left.concurrency) + rightConcurrency, rightErr := strconv.Atoi(right.concurrency) + if leftErr == nil && rightErr == nil && leftConcurrency != rightConcurrency { + return leftConcurrency < rightConcurrency + } + return left.concurrency < right.concurrency +} + +func modeOrder(mode string) int { + switch mode { + case "uncached": + return 0 + case "cached": + return 1 + default: + return 99 + } +} + +func percentile(values []float64, fraction float64) *float64 { + if len(values) == 0 { + return nil + } + ordered := append([]float64(nil), values...) + sort.Float64s(ordered) + position := float64(len(ordered)-1) * fraction + lower := int(position) + upper := min(lower+1, len(ordered)-1) + value := ordered[lower] + if lower != upper { + weight := position - float64(lower) + value = ordered[lower]*(1-weight) + ordered[upper]*weight + } + return &value +} + +func median(values []float64) *float64 { + if len(values) == 0 { + return nil + } + ordered := append([]float64(nil), values...) + sort.Float64s(ordered) + middle := len(ordered) / 2 + value := ordered[middle] + if len(ordered)%2 == 0 { + value = (ordered[middle-1] + ordered[middle]) / 2 + } + return &value +} + +func mean(values []float64) *float64 { + if len(values) == 0 { + return nil + } + var total float64 + for _, value := range values { + total += value + } + result := total / float64(len(values)) + return &result +} + +func formatRate(ok, total int) string { + if total == 0 { + return "-" + } + return fmt.Sprintf("%d/%d (%.0f%%)", ok, total, float64(ok)/float64(total)*100) +} + +func formatSeconds(value float64) string { + if value <= 0 { + return "-" + } + return fmt.Sprintf("%.2f", value) +} + +func formatRatePerSecond(count int, duration float64) string { + if count == 0 || duration == 0 { + return "-" + } + return fmt.Sprintf("%.1f", float64(count)/duration) +} + +func formatMilliseconds(value *float64, precision int) string { + if value == nil { + return "-" + } + return strconv.FormatFloat(*value*1000, 'f', precision, 64) +} + +func formatOptionalMilliseconds(value *float64, precision int) string { + return formatMilliseconds(value, precision) +} + +func formatOptionalFloat(value *float64, precision int) string { + if value == nil { + return "-" + } + return strconv.FormatFloat(*value, 'f', precision, 64) +} + +func formatSize(value *float64) string { + if value == nil { + return "-" + } + units := []string{"B", "KiB", "MiB", "GiB"} + size := *value + unit := units[0] + for _, candidate := range units { + unit = candidate + if math.Abs(size) < 1024 || candidate == units[len(units)-1] { + break + } + size /= 1024 + } + if unit == "B" { + return fmt.Sprintf("%.0f %s", size, unit) + } + return fmt.Sprintf("%.1f %s", size, unit) +} + +func atomicWrite(path string, content []byte) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat %s: %w", path, err) + } + temporary, err := os.CreateTemp(filepath.Dir(path), ".triplet-benchmark-*") + if err != nil { + return fmt.Errorf("create temporary file for %s: %w", path, err) + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(info.Mode().Perm()); err != nil { + _ = temporary.Close() + return fmt.Errorf("set permissions on temporary file for %s: %w", path, err) + } + if _, err := temporary.Write(content); err != nil { + _ = temporary.Close() + return fmt.Errorf("write temporary file for %s: %w", path, err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary file for %s: %w", path, err) + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("replace %s: %w", path, err) + } + return nil +} diff --git a/cmd/triplet-benchmark-helper/main_test.go b/cmd/triplet-benchmark-helper/main_test.go new file mode 100644 index 0000000..109dc29 --- /dev/null +++ b/cmd/triplet-benchmark-helper/main_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPercentEncode(t *testing.T) { + t.Parallel() + if got, want := percentEncode("a b/c+☃"), "a%20b%2Fc%2B%E2%98%83"; got != want { + t.Fatalf("percentEncode() = %q, want %q", got, want) + } +} + +func TestHashValuesUsesNULSeparators(t *testing.T) { + t.Parallel() + const want = "59b271ae1bbcb1d31d41929817f4b16fb439eb4f31520b5ad1d5ce98920a7138" + if got := hashValues([]string{"a", "b"}); got != want { + t.Fatalf("hashValues() = %q, want %q", got, want) + } +} + +func TestUpdateRun(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "run.json") + writeTestFile(t, path, `{"run_id":"test","concurrency":2}`+"\n") + + if err := updateRun(path, "start", "finish", "10.100000", "12.3456789"); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(string(content), "\n") { + t.Fatal("updated run metadata must end with a newline") + } + var run map[string]any + if err := json.Unmarshal(content, &run); err != nil { + t.Fatal(err) + } + if got := run["measured_started_at"]; got != "start" { + t.Fatalf("measured_started_at = %#v", got) + } + if got := run["measured_finished_at"]; got != "finish" { + t.Fatalf("measured_finished_at = %#v", got) + } + if got := run["measured_duration_seconds"]; got != 2.245679 { + t.Fatalf("measured_duration_seconds = %#v", got) + } +} + +func TestMatrixSummaryAndReportAppend(t *testing.T) { + t.Parallel() + root := t.TempDir() + index := filepath.Join(root, "matrix", "report.md") + if err := os.MkdirAll(filepath.Dir(index), 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, index, "# Benchmark Matrix: run\n\n## Matrix Runs\n\n| Mode |\n") + + runDir := filepath.Join(root, "run-uncached-c2") + if err := os.MkdirAll(runDir, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(runDir, "run.json"), `{"mode":"uncached","concurrency":2,"triplet_image":"triplet:test","measured_duration_seconds":2}`+"\n") + writeTestFile(t, filepath.Join(runDir, "requests.csv"), strings.Join([]string{ + "server,curl_exit,http_code,time_total,size_download", + "triplet,0,200,0.1,1024", + "triplet,0,201,0.3,2048", + "triplet,28,000,1.0,0", + }, "\n")+"\n") + writeTestFile(t, filepath.Join(runDir, "resource-summary.csv"), "server,mean_cpu_percent,max_mem_mib\ntriplet,50,12.5\n") + writeTestFile(t, filepath.Join(runDir, "report.md"), "# Run\n\n## Detail\n") + + if err := writeMatrixSummary(root, "run", index); err != nil { + t.Fatal(err) + } + content := readTestFile(t, index) + for _, expected := range []string{ + "Triplet image: `triplet:test`", + "| uncached | 2 | 2/3 (67%) | 2.00 | 1.0 | 290.0 | 298.0 | 500.00 | 12.5 |", + "## Matrix Runs", + "| uncached | 2 | triplet | 2/3 (67%) | 200.0 | 200.0 | 1.5 KiB | [report](../run-uncached-c2/report.md) |", + } { + if !strings.Contains(content, expected) { + t.Errorf("matrix report does not contain %q:\n%s", expected, content) + } + } + + if err := appendMatrixReports(root, "run", index); err != nil { + t.Fatal(err) + } + content = readTestFile(t, index) + for _, expected := range []string{ + "## Run Reports", + "### run-uncached-c2", + "- Mode: `uncached`", + "## Run\n\n### Detail", + } { + if !strings.Contains(content, expected) { + t.Errorf("appended report does not contain %q:\n%s", expected, content) + } + } +} + +func writeTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func readTestFile(t *testing.T, path string) string { + t.Helper() + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(content) +} diff --git a/cmd/triplet-conformance-check/main.go b/cmd/triplet-conformance-check/main.go new file mode 100644 index 0000000..44ab057 --- /dev/null +++ b/cmd/triplet-conformance-check/main.go @@ -0,0 +1,98 @@ +// Command triplet-conformance-check validates responses captured by the +// repository's HTTP conformance smoke test without embedding a scripting +// language in shell automation. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + + imageschema "github.com/libops/triplet/internal/iiif/image/v3/schema" + presentationvalidate "github.com/libops/triplet/internal/iiif/presentation/v3/validate" +) + +func main() { + infoPath := flag.String("info", "", "captured Image API info.json") + manifestPath := flag.String("manifest", "", "captured Presentation API Manifest") + annotationPagePath := flag.String("annotation-page", "", "captured Presentation API AnnotationPage") + flag.Parse() + if err := run(*infoPath, *manifestPath, *annotationPagePath); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(infoPath, manifestPath, annotationPagePath string) error { + if infoPath == "" || manifestPath == "" || annotationPagePath == "" { + return errors.New("-info, -manifest, and -annotation-page are required") + } + info, err := os.ReadFile(infoPath) + if err != nil { + return fmt.Errorf("read info.json: %w", err) + } + if err := validateInfo(info); err != nil { + return err + } + manifest, err := os.ReadFile(manifestPath) + if err != nil { + return fmt.Errorf("read manifest: %w", err) + } + if err := validateManifest(manifest); err != nil { + return err + } + page, err := os.ReadFile(annotationPagePath) + if err != nil { + return fmt.Errorf("read annotation page: %w", err) + } + return validateAnnotationPage(page) +} + +func validateInfo(body []byte) error { + if err := imageschema.ValidateInfoBytes(body); err != nil { + return fmt.Errorf("validate info.json: %w", err) + } + return nil +} + +func validateManifest(body []byte) error { + resource, err := presentationvalidate.ValidateResourceBytes(body) + if err != nil { + return fmt.Errorf("validate manifest: %w", err) + } + if resource.Type != "Manifest" { + return fmt.Errorf("manifest resource type is %q", resource.Type) + } + var document struct { + Items []json.RawMessage `json:"items"` + } + if err := json.Unmarshal(body, &document); err != nil { + return fmt.Errorf("decode manifest: %w", err) + } + if len(document.Items) == 0 { + return errors.New("manifest items must not be empty") + } + return nil +} + +func validateAnnotationPage(body []byte) error { + resource, err := presentationvalidate.ValidateResourceBytes(body) + if err != nil { + return fmt.Errorf("validate annotation page: %w", err) + } + if resource.Type != "AnnotationPage" { + return fmt.Errorf("annotation page resource type is %q", resource.Type) + } + var document struct { + Items []json.RawMessage `json:"items"` + } + if err := json.Unmarshal(body, &document); err != nil { + return fmt.Errorf("decode annotation page: %w", err) + } + if len(document.Items) == 0 { + return errors.New("annotation page items must not be empty") + } + return nil +} diff --git a/cmd/triplet-conformance-check/main_test.go b/cmd/triplet-conformance-check/main_test.go new file mode 100644 index 0000000..c1afc65 --- /dev/null +++ b/cmd/triplet-conformance-check/main_test.go @@ -0,0 +1,25 @@ +package main + +import "testing" + +func TestValidatePresentationFixtures(t *testing.T) { + manifest := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/manifest","type":"Manifest","label":{"en":["Manifest"]},"items":[{"id":"https://example.org/canvas/1","type":"Canvas","width":100,"height":100,"items":[]}]}`) + if err := validateManifest(manifest); err != nil { + t.Fatalf("valid manifest rejected: %v", err) + } + page := []byte(`{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"https://example.org/page/1","type":"AnnotationPage","items":[{"id":"https://example.org/annotation/1","type":"Annotation","target":"https://example.org/canvas/1","textGranularity":"line"}]}`) + if err := validateAnnotationPage(page); err != nil { + t.Fatalf("valid annotation page rejected: %v", err) + } +} + +func TestValidatePresentationFixturesRejectsEmptyResources(t *testing.T) { + manifest := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/manifest","type":"Manifest","label":{"en":["Manifest"]},"items":[]}`) + if err := validateManifest(manifest); err == nil { + t.Fatal("empty manifest accepted") + } + page := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/page/1","type":"AnnotationPage","items":[]}`) + if err := validateAnnotationPage(page); err == nil { + t.Fatal("empty annotation page accepted") + } +} diff --git a/config.example.yaml b/config.example.yaml index 521a34e..230aa9c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -119,13 +119,14 @@ iiif: enabled: false prefix: /presentation/v3 # Configure exactly one backend: root for filesystem, or dsn for MariaDB. - # Filesystem-backed presentation store. Manifests live at - # {root}/{itemID}/manifest.json. Annotation pages live at - # {root}/{itemID}/canvas/{canvasID}/annotations.json. + # The filesystem backend hashes normalized public resource paths beneath + # this root and is intended for a single Triplet process. Populate + # resources through the conditional HTTP write API; the on-disk layout is + # an implementation detail. Use MariaDB for replicated deployments. root: ./testdata/presentation - # dsn: scribe:scribe@tcp(mariadb:3306)/scribe?parseTime=true - # Writes are disabled unless explicitly enabled and authenticated with a - # Bearer token. Prefer injecting this from the environment. + # dsn: triplet:triplet@tcp(mariadb:3306)/triplet?parseTime=true + # PUT and DELETE are disabled unless explicitly enabled and authenticated + # with a Bearer token. Prefer injecting this from the environment. write_enabled: false # write_token: ${TRIPLET_PRESENTATION_WRITE_TOKEN} @@ -171,6 +172,12 @@ sources: # http: # allowed_origins: [https://islandora-stage.lib.lehigh.edu] # allow_private_hosts: false + # # Opt in only when the source origin authorizes each caller. Triplet + # # forwards only Cookie and Authorization, never arbitrary request headers. + # # Cross-origin redirects never receive those credentials. Exact + # # allowed_origins remain mandatory. Shared source and metadata caches + # # must be disabled when this is true. + # forward_auth_headers: false # request_timeout: 2m # max_bytes: 50MiB # # Optional in-process metadata cache for remote URL identifiers. This lets diff --git a/deploy/compose/README.md b/deploy/compose/README.md index f150e6e..8bc44cb 100644 --- a/deploy/compose/README.md +++ b/deploy/compose/README.md @@ -1,8 +1,13 @@ # Self-host with docker-compose -Single-host deployment of triplet. Source images come from the local -`./images/` directory mounted into the container. Presentation manifests and -annotation pages come from `./presentation/`. +Single-host deployment of Triplet. Source images come from the local +`./images/` directory mounted read-only into the container. Byte-exact, +path-keyed Presentation resources and derivative caches use Docker named +volumes. A one-shot, network-disabled initializer grants the rootless Triplet +user ownership of only those three writable volume roots before the server +starts. The server itself still runs without root or Linux capabilities on a +read-only root filesystem, and host directories do not need world-writable +permissions. ```sh mkdir -p images @@ -36,13 +41,16 @@ Then: curl http://localhost:8080/iiif/3/sample.png/info.json ``` -Presentation writes are disabled by default. To exercise the writable -annotation route and conformance write checks: +Presentation writes are disabled by default. To exercise generic conditional +resource writes and conformance checks: ```sh TRIPLET_PRESENTATION_WRITE_ENABLED=true TRIPLET_PRESENTATION_WRITE_TOKEN=dev-token docker compose up TRIPLET_PRESENTATION_WRITE_TOKEN=dev-token ../../scripts/conformance.sh ``` +Back up the `presentation` named volume as durable application data. The +`cache` and `source-cache` volumes contain rebuildable derivatives. + For TLS, front this with nginx, Caddy, or your reverse proxy of choice — the triplet container does not terminate TLS itself. diff --git a/deploy/compose/docker-compose.yaml b/deploy/compose/docker-compose.yaml index 8638885..d8ccc8c 100644 --- a/deploy/compose/docker-compose.yaml +++ b/deploy/compose/docker-compose.yaml @@ -4,10 +4,10 @@ services: profiles: - integration environment: - MARIADB_DATABASE: ${MARIADB_DATABASE:-scribe} - MARIADB_USER: ${MARIADB_USER:-scribe} - MARIADB_PASSWORD: ${MARIADB_PASSWORD:-scribe} - MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD:-scribe-root} + MARIADB_DATABASE: ${MARIADB_DATABASE:-triplet} + MARIADB_USER: ${MARIADB_USER:-triplet} + MARIADB_PASSWORD: ${MARIADB_PASSWORD:-triplet} + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD:-triplet-root} volumes: - mariadb:/var/lib/mysql healthcheck: @@ -17,19 +17,49 @@ services: retries: 30 restart: unless-stopped + triplet-volume-init: + image: ${TRIPLET_IMAGE:-ghcr.io/libops/triplet:${GIT_BRANCH:-main}} + user: "0:0" + entrypoint: + - /bin/chown + command: + - triplet:triplet + - /var/lib/triplet/presentation + - /var/lib/triplet/cache + - /var/lib/triplet/source-cache + volumes: + - presentation:/var/lib/triplet/presentation + - cache:/var/lib/triplet/cache + - source-cache:/var/lib/triplet/source-cache + network_mode: none + read_only: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + cap_add: + - CHOWN + restart: "no" + triplet: image: ${TRIPLET_IMAGE:-ghcr.io/libops/triplet:${GIT_BRANCH:-main}} # Or build from the repo root: # build: # context: ../.. + depends_on: + triplet-volume-init: + condition: service_completed_successfully ports: - "8080:8080" + environment: + TRIPLET_PRESENTATION_WRITE_ENABLED: ${TRIPLET_PRESENTATION_WRITE_ENABLED:-false} + TRIPLET_PRESENTATION_WRITE_TOKEN: ${TRIPLET_PRESENTATION_WRITE_TOKEN:-} volumes: - ./config.yaml:/etc/triplet/config.yaml:ro - ./images:/var/lib/triplet/images:ro - - ./presentation:/var/lib/triplet/presentation - - ./cache:/var/lib/triplet/cache - - ./source-cache:/var/lib/triplet/source-cache + - presentation:/var/lib/triplet/presentation + - cache:/var/lib/triplet/cache + - source-cache:/var/lib/triplet/source-cache # Optional private CA bundle for upstream HTTPS sources. Concatenate one # or more PEM-encoded CA certs into this file; Go's TLS verifier reads # PEM files directly from /etc/ssl/certs. @@ -51,3 +81,6 @@ services: volumes: mariadb: + presentation: + cache: + source-cache: diff --git a/deploy/compose/presentation/item-1/canvas/canvas-1/annotations.json b/deploy/compose/presentation/resources/54/54b1ff4c782cca754695b23794617d7b76c96600ee21f44f2cb1741160c48e40.json similarity index 100% rename from deploy/compose/presentation/item-1/canvas/canvas-1/annotations.json rename to deploy/compose/presentation/resources/54/54b1ff4c782cca754695b23794617d7b76c96600ee21f44f2cb1741160c48e40.json diff --git a/deploy/compose/presentation/item-1/manifest.json b/deploy/compose/presentation/resources/cb/cbf1fbf4541d61625920ce657218bb5661d505b93c01e636e245b66f34062db3.json similarity index 95% rename from deploy/compose/presentation/item-1/manifest.json rename to deploy/compose/presentation/resources/cb/cbf1fbf4541d61625920ce657218bb5661d505b93c01e636e245b66f34062db3.json index 55e20e2..9ea4a50 100644 --- a/deploy/compose/presentation/item-1/manifest.json +++ b/deploy/compose/presentation/resources/cb/cbf1fbf4541d61625920ce657218bb5661d505b93c01e636e245b66f34062db3.json @@ -23,7 +23,8 @@ "id": "http://localhost:8080/presentation/v3/item-1/canvas/canvas-1/annotations", "type": "AnnotationPage" } - ] + ], + "items": [] } ] } diff --git a/deploy/seccomp-triplet.json b/deploy/seccomp-triplet.json index dfd83a6..799b310 100644 --- a/deploy/seccomp-triplet.json +++ b/deploy/seccomp-triplet.json @@ -27,8 +27,10 @@ "exit_group", "faccessat2", "fcntl", + "fchmod", "fstat", "fstatfs", + "fsync", "futex", "getcwd", "getdents64", @@ -40,6 +42,7 @@ "lseek", "madvise", "mmap", + "mkdirat", "mprotect", "munmap", "nanosleep", @@ -52,6 +55,7 @@ "readlinkat", "recvfrom", "recvmsg", + "renameat", "restart_syscall", "rt_sigaction", "rt_sigprocmask", @@ -68,6 +72,7 @@ "statfs", "statx", "tgkill", + "unlinkat", "write" ], "action": "SCMP_ACT_ALLOW" diff --git a/docs/authorization.md b/docs/authorization.md index 3f550cd..392fdba 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -184,6 +184,10 @@ loopback, link-local, and metadata addresses are blocked unless `sources.http.allow_private_hosts` is explicitly enabled. When private hosts are blocked, Triplet resolves the hostname once and connects only to a verified public IP, so DNS rebinding cannot swap the connection target after validation. +Shared, benchmarking, documentation, and reserved address ranges are treated as +non-public. Source identifiers containing URL userinfo credentials are rejected; +caller credentials can enter this boundary only through the explicit forwarding +option below. ```yaml sources: @@ -191,6 +195,28 @@ sources: allowed_origins: - https://repository.example.edu allow_private_hosts: false + forward_auth_headers: false request_timeout: 2m max_bytes: 50MiB ``` + +### Protected HTTP sources + +For an ordinary HTTP source that authorizes each caller, opt in explicitly: + +```yaml +sources: + default: http + http: + allowed_origins: + - https://repository.example.edu + forward_auth_headers: true + metadata_cache_ttl: 0 +``` + +Triplet copies only `Cookie` and `Authorization` from the incoming image +request. The initial target and every redirect must be an exact allowed origin; +credentials are removed on cross-origin redirects even when the destination is +also allowed. Shared source-byte and metadata caches are incompatible with this +mode and configuration validation rejects them. The Image handler still checks +the protected source before it can serve a shared derivative-cache hit. diff --git a/docs/caching.md b/docs/caching.md index 939eefc..eb64149 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -103,6 +103,12 @@ cache: When `source_stale_after` is set, stale hits are served immediately and refreshed in the background. Upstream 4xx/5xx responses are not stored. +The source cache cannot be enabled with +`sources.http.forward_auth_headers: true`. Authorization and cookie values are +intentionally not cache-key material, so sharing cached source bytes between +credentialed requests could disclose a protected source. Triplet rejects this +combination during configuration validation. + ## HTTP metadata cache Remote URL identifiers need source metadata to build derivative cache keys. By @@ -121,6 +127,11 @@ cache hit can be served without touching the remote source. If the remote source changes, disappears, or changes authorization during the TTL, Triplet may serve the cached derivative until the metadata entry expires. +The metadata cache likewise cannot be enabled when +`sources.http.forward_auth_headers` is true. With forwarding enabled, Triplet +reauthorizes every image request against its exact-allowlisted source origin +before it can serve a derivative cache hit. + ## Authorization decision cache Local URL mappings with `auth_probe: true` cache anonymous and credentialed @@ -165,8 +176,8 @@ derivative and source caches. | Layer | Configuration | What is cached | Invalidation / freshness | |---|---|---|---| | Derivative cache | `cache.root`; optional `cache.max_bytes`, `cache.max_age`, `iiif.image.cache_invalidation_token` | Encoded IIIF image responses, keyed by identifier, source version, invalidation marker, region, size, rotation, quality, and format. | A changed source version produces a new key. The protected invalidation route bumps the per-identifier invalidation marker. `cache.max_bytes` is a best-effort aggregate cache budget reported by `triplet-cache-cleanup`; `cache.max_age` is enforced on reads and by `triplet-cache-cleanup`. `iiif.image.max_derivative_bytes` is the per-response size limit before return/cache. Failed transforms and HTTP error responses are not stored. | -| HTTP source cache | `cache.source_root`; optional `cache.source_max_bytes`, `cache.source_stale_after` | Original source bytes fetched through the HTTP source backend. | Keys are source identifiers. When `source_stale_after` is set, stale hits are served immediately and refreshed in the background. Upstream 4xx/5xx responses are not stored. | -| HTTP metadata cache | `sources.http.metadata_cache_ttl` | Successful remote source metadata lookups for URL identifiers. | In-memory only. While fresh, derivative cache checks can avoid upstream metadata requests. This can serve stale derivatives until the TTL expires. | +| HTTP source cache | `cache.source_root`; optional `cache.source_max_bytes`, `cache.source_stale_after` | Original source bytes fetched through the HTTP source backend. | Keys are source identifiers. When `source_stale_after` is set, stale hits are served immediately and refreshed in the background. Upstream 4xx/5xx responses are not stored. Incompatible with `sources.http.forward_auth_headers`. | +| HTTP metadata cache | `sources.http.metadata_cache_ttl` | Successful remote source metadata lookups for URL identifiers. | In-memory only. While fresh, derivative cache checks can avoid upstream metadata requests. This can serve stale derivatives until the TTL expires. Incompatible with `sources.http.forward_auth_headers`. | | `info.json` dimension cache | `iiif.image.info_dimension_cache` | Source dimensions used to build Image API `info.json`. | In-memory only. Entries are keyed by identifier plus source size/modtime metadata, so source changes with updated metadata miss the cache. | | Local URL auth-probe cache | `sources.http.metadata_cache_ttl` for mappings with `auth_probe: true` | Authorization probe results for local URL mappings. Anonymous and credentialed probes are cached separately. See [Authorization](authorization.md). | In-memory only. The image cache invalidation route also clears matching auth-probe entries when the source backend supports it. | | libvips operation cache | `vips.cache_max_mem`, `vips.cache_max_files` | libvips in-process operation results. | Disabled by default in the example config. This is process-local and separate from Triplet's derivative/source caches. | diff --git a/docs/configuration.md b/docs/configuration.md index f31ca5e..9c1545a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,8 +61,54 @@ iiif: ``` When Presentation is enabled, Triplet exposes `ETag` through CORS so browser -annotation editors can read it and send optimistic-concurrency writes with -`If-Match`. +clients can perform conditional reads and optimistic-concurrency writes. It +also exposes `Last-Modified`, `Content-Length`, and `Location`. + +## Presentation resources + +Triplet stores byte-exact Presentation resources at arbitrary normalized paths +beneath `iiif.presentation.prefix`. For example, a deployment can choose paths +such as `/presentation/v3/items/123/manifest` and +`/presentation/v3/items/123/canvas/1/annotations`; Triplet does not impose an +application identifier or tenancy scheme. + +The JSON-LD `id` of every stored resource must exactly equal the public URL +built from `server.public_base_url`, the Presentation prefix, and its canonical +path. Query strings, path traversal, encoded separators, and non-canonical +escaping are rejected. Supported top-level resources are `Manifest`, `Canvas`, +`Collection`, `AnnotationCollection`, `AnnotationPage`, and `Annotation`. + +`GET` and `HEAD` return strong `ETag` and `Last-Modified` validators. They +honor `If-None-Match` and `If-Modified-Since`. When writes are enabled, a Bearer +token protects both mutation methods: + +- Create with `PUT` and `If-None-Match: *`; success returns `201 Created` and + `Location`. +- Replace with `PUT` and a current strong `If-Match` value (or `*` when any + existing representation is acceptable); success returns `204 No Content`. +- Delete with `DELETE` and a current strong `If-Match` value (or `*`); success + returns `204 No Content`. + +Missing conditions return `428 Precondition Required`; stale conditions return +`412 Precondition Failed`. PUT accepts `application/json` and +`application/ld+json` bodies up to 8 MiB. + +Configure one Presentation backend. The filesystem layout is private and +hashes public paths to avoid path and filename ambiguity. Its compare-and-swap +lock is process-local, so use it for a single Triplet process only. Replicated +deployments use the MariaDB backend, whose transactional compare-and-swap uses +one generic byte-preserving resource table without foreign keys. + +```yaml +iiif: + presentation: + enabled: true + prefix: /presentation/v3 + root: ./testdata/presentation + # dsn: triplet:triplet@tcp(mariadb:3306)/triplet?parseTime=true + write_enabled: true + write_token: ${TRIPLET_PRESENTATION_WRITE_TOKEN} +``` ## Image safety limits @@ -167,6 +213,7 @@ sources: allowed_origins: - https://repository.example.edu allow_private_hosts: false + forward_auth_headers: false request_timeout: 2m max_bytes: 50MiB ``` @@ -219,6 +266,18 @@ sources: metadata_cache_ttl: 5m ``` +Set `forward_auth_headers: true` only when an exact allowed source origin +authorizes the caller that requested the IIIF image. Triplet then forwards only +the incoming `Cookie` and `Authorization` fields; it never forwards arbitrary +headers, and it strips both credentials before a cross-origin redirect. Use +HTTPS or an equivalently protected internal transport. + +Shared source-byte caching (`cache.source_root`) and shared metadata caching +(`sources.http.metadata_cache_ttl`) are rejected in this mode. Triplet contacts +the source on every request before serving a derivative-cache hit, so an +authorized caller cannot populate a source or metadata result that an +anonymous caller can reuse without authorization. + The HTTP host allowlist is a source-fetch boundary. See [Authorization](authorization.md#source-fetch-boundary) for origin, redirect, private host, and DNS rebinding behavior. diff --git a/docs/conformance.md b/docs/conformance.md index d47314e..9efb22b 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -9,13 +9,20 @@ For Go code, Triplet imports: - `github.com/libops/iiif-spec/image/v3/gen` - `github.com/libops/iiif-spec/image/v3/schema` - `github.com/libops/iiif-spec/presentation/v3/gen/...` +- `github.com/libops/iiif-spec/presentation/v3/schema` +- `github.com/libops/iiif-spec/extension/textgranularity/schema` Triplet's local `types/` packages are thin aliases or wrappers on top of those imported wire types where the server needs stable names or extension fields beyond the upstream schemas. -Triplet also tracks extension support in code and tests. In particular, the -Presentation annotation path validates the IIIF Text Granularity extension. +Presentation writes and reads use the extension-aware validators from +`iiif-spec`, including its standalone Canvas validation. This keeps legal +JSON-LD extension properties byte-exact while still validating core properties +and requiring the Presentation context to be the final top-level context. +Annotation pages and standalone annotations additionally use the generic IIIF +Text Granularity schema. Application-specific OCR profiles belong in the +application, not in Triplet. The IIIF API surfaces are configured independently: @@ -28,9 +35,13 @@ iiif: enabled: false prefix: /presentation/v3 root: ./testdata/presentation - # dsn: scribe:scribe@tcp(mariadb:3306)/scribe?parseTime=true + # dsn: triplet:triplet@tcp(mariadb:3306)/triplet?parseTime=true write_enabled: false ``` -Presentation annotation writes use strong ETags and require `If-Match`, and the -Presentation CORS policy exposes `ETag` for browser-based annotation editors. +Presentation supports path-keyed `Manifest`, `Canvas`, `Collection`, +`AnnotationCollection`, `AnnotationPage`, and `Annotation` resources. It +preserves request bytes exactly and validates each representation before both +storage and delivery. Create uses `If-None-Match: *`; replace and delete use a +strong `If-Match`. Conditional GET/HEAD, complete mutation CORS headers, and +route/body `id` coherence are covered by handler and store contract tests. diff --git a/docs/deploys.md b/docs/deploys.md index a738ce3..c75c8aa 100644 --- a/docs/deploys.md +++ b/docs/deploys.md @@ -27,7 +27,7 @@ iiif: presentation: enabled: true prefix: /presentation/v3 - dsn: scribe:scribe@tcp(mariadb:3306)/scribe?parseTime=true + dsn: triplet:triplet@tcp(mariadb:3306)/triplet?parseTime=true write_enabled: false ``` @@ -36,4 +36,9 @@ triplet -config config.yaml -migrate-presentation-mariadb ``` Normal server startup does not run DDL, so the runtime DSN can use a -least-privilege account after migration. +least-privilege account after migration. The schema contains one generic +path-keyed resource table with byte-preserving bodies and no foreign keys. + +Treat the Presentation root or MariaDB table as durable state unless the +publishing application explicitly documents it as a rebuildable projection. +Back up and restore it consistently with that application's publication state. diff --git a/go.mod b/go.mod index 8feea77..9b88ef8 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.26.2 require ( github.com/davidbyttow/govips/v2 v2.18.0 github.com/go-sql-driver/mysql v1.10.0 - github.com/libops/iiif-spec v0.1.0 - github.com/prometheus/client_golang v1.23.2 + github.com/libops/iiif-spec v0.3.0 + github.com/prometheus/client_golang v1.24.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -19,15 +19,13 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.20.1 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/image v0.43.0 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/image v0.44.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 9769cff..94d3baa 100644 --- a/go.sum +++ b/go.sum @@ -13,32 +13,30 @@ github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxo github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/libops/govips/v2 v2.0.0-20260429094915-ee9c36250135 h1:AGWc7KZgVO1MuqZl3BCW4R/b0h8lShHkA4nqQhJtlOk= github.com/libops/govips/v2 v2.0.0-20260429094915-ee9c36250135/go.mod h1:8+nst5zfMoats12PgmmAPh6p5OfjDaXK0BXMFl/vOcM= -github.com/libops/iiif-spec v0.1.0 h1:qiEokzXE3F8ZFAarA0Q0Ai6dqtndzNRt2mBQj8EvQGs= -github.com/libops/iiif-spec v0.1.0/go.mod h1:0HuD9gEQePZ6+z9SyH9TJoMRf9l1af02DyKQsMgQebU= +github.com/libops/iiif-spec v0.3.0 h1:+sQNSlJnG0oV1TDsap/sHWC6RAAjrlP7bvYcDlvBI48= +github.com/libops/iiif-spec v0.3.0/go.mod h1:0HuD9gEQePZ6+z9SyH9TJoMRf9l1af02DyKQsMgQebU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= +github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -47,18 +45,18 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY= -golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= +golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go index 150917e..307ea90 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -136,11 +136,15 @@ type FileURLMapping struct { // HTTPSource resolves identifiers that are HTTP(S) URLs. type HTTPSource struct { - AllowedOrigins []string `yaml:"allowed_origins"` - AllowPrivateHosts bool `yaml:"allow_private_hosts"` - RequestTimeout time.Duration `yaml:"request_timeout"` - MaxBytes ByteSize `yaml:"max_bytes"` - MetadataCacheTTL time.Duration `yaml:"metadata_cache_ttl"` + AllowedOrigins []string `yaml:"allowed_origins"` + AllowPrivateHosts bool `yaml:"allow_private_hosts"` + // ForwardAuthHeaders forwards only Cookie and Authorization from the + // incoming image request to exact allowed origins. It is opt-in because + // those values are caller credentials. + ForwardAuthHeaders bool `yaml:"forward_auth_headers"` + RequestTimeout time.Duration `yaml:"request_timeout"` + MaxBytes ByteSize `yaml:"max_bytes"` + MetadataCacheTTL time.Duration `yaml:"metadata_cache_ttl"` } // Cache declares optional derivative-cache settings. @@ -333,7 +337,7 @@ func (c *Config) validate() error { if err != nil { return fmt.Errorf("server.public_base_url: %w", err) } - if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil { return fmt.Errorf("server.public_base_url: must be an absolute http(s) URL, got %q", c.Server.PublicBaseURL) } if u.RawQuery != "" || u.Fragment != "" { @@ -453,6 +457,12 @@ func (c *Config) validate() error { if c.Sources.HTTP.MetadataCacheTTL < 0 { return errors.New("sources.http.metadata_cache_ttl: must be >= 0") } + if c.Sources.HTTP.ForwardAuthHeaders && c.Sources.HTTP.MetadataCacheTTL > 0 { + return errors.New("sources.http.metadata_cache_ttl must be 0 when sources.http.forward_auth_headers = true") + } + if c.Sources.HTTP.ForwardAuthHeaders && c.Cache.SourceRoot != "" { + return errors.New("cache.source_root must be empty when sources.http.forward_auth_headers = true") + } } if c.Sources.File != nil { for _, prefix := range c.Sources.File.URLPrefixes { @@ -519,7 +529,7 @@ func validateAllowedOrigins(name string, origins []string) error { return fmt.Errorf("%s: origin %q must not contain whitespace", name, origin) } u, err := url.Parse(origin) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { return fmt.Errorf("%s: origin %q must be an absolute http(s) origin or *", name, origin) } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5325d3a..2e86f79 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -69,6 +69,18 @@ sources: default: file file: root: /tmp +`, + wantErr: "server.public_base_url", + }, + { + name: "public_base_url credentials rejected", + body: ` +server: + public_base_url: https://user:secret@example.org/iiif +sources: + default: file + file: + root: /tmp `, wantErr: "server.public_base_url", }, @@ -98,6 +110,59 @@ sources: metadata_cache_ttl: 24h `, }, + { + name: "http source auth forwarding valid", + body: ` +server: + public_base_url: http://localhost:8080 +sources: + default: http + http: + allowed_origins: [https://repository.example.org] + forward_auth_headers: true +`, + }, + { + name: "http source origin rejects URL credentials", + body: ` +server: + public_base_url: http://localhost:8080 +sources: + default: http + http: + allowed_origins: [https://user:secret@repository.example.org] +`, + wantErr: "absolute http(s) origin", + }, + { + name: "http auth forwarding rejects shared metadata cache", + body: ` +server: + public_base_url: http://localhost:8080 +sources: + default: http + http: + allowed_origins: [https://repository.example.org] + forward_auth_headers: true + metadata_cache_ttl: 5m +`, + wantErr: "metadata_cache_ttl must be 0", + }, + { + name: "http auth forwarding rejects shared source cache", + body: ` +server: + public_base_url: http://localhost:8080 +sources: + default: http + http: + allowed_origins: [https://repository.example.org] + forward_auth_headers: true +cache: + source_root: /tmp/source-cache +`, + wantErr: "cache.source_root must be empty", + }, { name: "file url prefixes valid", body: ` @@ -341,7 +406,7 @@ iiif: presentation: enabled: true root: /tmp - dsn: scribe:scribe@tcp(mariadb:3306)/scribe?parseTime=true + dsn: triplet:triplet@tcp(mariadb:3306)/triplet?parseTime=true sources: default: file file: @@ -578,6 +643,25 @@ sources: } } +func TestLoadHTTPForwardAuthHeaders(t *testing.T) { + path := writeConfig(t, ` +server: + public_base_url: http://localhost:8080 +sources: + default: http + http: + allowed_origins: [https://repository.example.org] + forward_auth_headers: true +`) + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Sources.HTTP == nil || !cfg.Sources.HTTP.ForwardAuthHeaders { + t.Fatalf("HTTP source = %#v", cfg.Sources.HTTP) + } +} + func TestLoadExpandsEnvironmentVariables(t *testing.T) { t.Setenv("TRIPLET_PUBLIC_BASE_URL", "https://iiif.example.org") t.Setenv("TRIPLET_IMAGE_ROOT", "/srv/images") diff --git a/internal/contracts/repository_contracts_test.go b/internal/contracts/repository_contracts_test.go new file mode 100644 index 0000000..175ab5a --- /dev/null +++ b/internal/contracts/repository_contracts_test.go @@ -0,0 +1,264 @@ +package contracts + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +var pythonInvocation = regexp.MustCompile(`(^|[[:space:];|&(!])("[^"]*/python([0-9.]*)?"|'[^']*/python([0-9.]*)?'|[^[:space:];|&()'"]*python([0-9.]*))[[:space:]]+(.*)$`) + +func TestShellScriptsDoNotInlinePython(t *testing.T) { + t.Parallel() + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve contract test source") + } + root := filepath.Clean(filepath.Join(filepath.Dir(source), "..", "..")) + + var violations []string + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + switch entry.Name() { + case ".cache", ".git", "node_modules", "results", "site", "vendor": + if path != root { + return filepath.SkipDir + } + } + return nil + } + if filepath.Ext(path) != ".sh" { + return nil + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + for _, violation := range inlinePythonLines(string(content)) { + violations = append(violations, fmt.Sprintf("%s:%s", strings.TrimPrefix(path, root+string(filepath.Separator)), violation)) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(violations) > 0 { + t.Fatalf("shell scripts may invoke packaged Python tools but may not use Python -c, stdin, or heredocs:\n%s", strings.Join(violations, "\n")) + } +} + +func TestInlinePythonLines(t *testing.T) { + t.Parallel() + tests := []struct { + name string + script string + violations int + }{ + {name: "packaged script", script: `python3 scripts/report.py input.csv`, violations: 0}, + {name: "module", script: `"$VENV_DIR/bin/python" -m pip install package`, violations: 0}, + {name: "command string", script: `python3 -c 'print(1)'`, violations: 1}, + {name: "attached command string", script: `python3 -cprint(1)`, violations: 1}, + {name: "tabbed command string", script: "python3\t-c\t'print(1)'", violations: 1}, + {name: "standard input", script: `python3 - "$file"`, violations: 1}, + {name: "heredoc", script: "python3 script.py <<'PY'\nvalue = 1\nPY", violations: 1}, + {name: "continued", script: "python3 \\\n -c 'print(1)'", violations: 1}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if got := len(inlinePythonLines(test.script)); got != test.violations { + t.Fatalf("inlinePythonLines() found %d violations, want %d", got, test.violations) + } + }) + } +} + +func TestConformanceOwnsAllTemporaryFiles(t *testing.T) { + t.Parallel() + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve contract test source") + } + scriptPath := filepath.Join(filepath.Dir(source), "..", "..", "scripts", "conformance.sh") + content, err := os.ReadFile(scriptPath) + if err != nil { + t.Fatal(err) + } + script := string(content) + if strings.Contains(script, "${TMPDIR}") || strings.Contains(script, "$TMPDIR/") { + t.Fatal("conformance script must not depend on an inherited TMPDIR") + } + for _, name := range []string{"write-annotations.json", "write-annotations-updated.json"} { + want := "${CONFORMANCE_TMP_DIR}/" + name + if !strings.Contains(script, want) { + t.Fatalf("conformance temporary artifact %q must be stored in the owned temporary directory", name) + } + } +} + +func TestComposePassesPresentationWriteEnvironment(t *testing.T) { + t.Parallel() + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve contract test source") + } + root := filepath.Clean(filepath.Join(filepath.Dir(source), "..", "..")) + composeBody, err := os.ReadFile(filepath.Join(root, "deploy", "compose", "docker-compose.yaml")) + if err != nil { + t.Fatal(err) + } + var compose struct { + Services map[string]struct { + Environment map[string]string `yaml:"environment"` + Volumes []string `yaml:"volumes"` + User string `yaml:"user"` + Command []string `yaml:"command"` + CapAdd []string `yaml:"cap_add"` + DependsOn map[string]struct { + Condition string `yaml:"condition"` + } `yaml:"depends_on"` + } `yaml:"services"` + Volumes map[string]any `yaml:"volumes"` + } + if err := yaml.Unmarshal(composeBody, &compose); err != nil { + t.Fatalf("parse Compose file: %v", err) + } + environment := compose.Services["triplet"].Environment + for _, name := range []string{"TRIPLET_PRESENTATION_WRITE_ENABLED", "TRIPLET_PRESENTATION_WRITE_TOKEN"} { + if _, ok := environment[name]; !ok { + t.Fatalf("Triplet Compose service must pass %s into the config-expansion environment", name) + } + } + tripletService := compose.Services["triplet"] + mounts := tripletService.Volumes + initService, ok := compose.Services["triplet-volume-init"] + if !ok { + t.Fatal("Compose must initialize writable volume ownership before Triplet starts") + } + if initService.User != "0:0" || !containsExact(initService.CapAdd, "CHOWN") { + t.Fatal("volume initializer must run with only the ownership capability it needs") + } + if len(initService.Command) == 0 || initService.Command[0] != "triplet:triplet" { + t.Fatal("volume initializer must assign writable roots to the rootless Triplet UID and GID") + } + dependency, ok := tripletService.DependsOn["triplet-volume-init"] + if !ok || dependency.Condition != "service_completed_successfully" { + t.Fatal("Triplet must wait for successful writable-volume initialization") + } + for _, name := range []string{"presentation", "cache", "source-cache"} { + if _, ok := compose.Volumes[name]; !ok { + t.Fatalf("Compose must declare the rootless-writable %s named volume", name) + } + want := name + ":/var/lib/triplet/" + name + if !containsExact(mounts, want) { + t.Fatalf("Triplet Compose service must mount %q", want) + } + if !containsExact(initService.Volumes, want) { + t.Fatalf("volume initializer must mount %q", want) + } + if !containsExact(initService.Command, "/var/lib/triplet/"+name) { + t.Fatalf("volume initializer must assign ownership of /var/lib/triplet/%s", name) + } + } + dockerfile, err := os.ReadFile(filepath.Join(root, "Dockerfile")) + if err != nil { + t.Fatal(err) + } + dockerfileText := string(dockerfile) + for _, path := range []string{"/var/lib/triplet/presentation", "/var/lib/triplet/cache", "/var/lib/triplet/source-cache"} { + if !strings.Contains(dockerfileText, path) { + t.Fatalf("runtime image must pre-create %s before Docker initializes its named volume", path) + } + } + if !strings.Contains(dockerfileText, "chown -R triplet:triplet /var/lib/triplet") { + t.Fatal("runtime image must make named-volume roots writable by the rootless Triplet user") + } +} + +func TestRuntimeSeccompAllowsConfiguredFileStores(t *testing.T) { + t.Parallel() + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve contract test source") + } + profileBody, err := os.ReadFile(filepath.Join(filepath.Dir(source), "..", "..", "deploy", "seccomp-triplet.json")) + if err != nil { + t.Fatal(err) + } + var profile struct { + DefaultAction string `json:"defaultAction"` + Syscalls []struct { + Names []string `json:"names"` + Action string `json:"action"` + } `json:"syscalls"` + } + if err := json.Unmarshal(profileBody, &profile); err != nil { + t.Fatalf("parse seccomp profile: %v", err) + } + if profile.DefaultAction != "SCMP_ACT_ERRNO" { + t.Fatalf("seccomp default action = %q, want SCMP_ACT_ERRNO", profile.DefaultAction) + } + allowed := make(map[string]bool) + for _, rule := range profile.Syscalls { + if rule.Action != "SCMP_ACT_ALLOW" { + continue + } + for _, name := range rule.Names { + allowed[name] = true + } + } + // The configured filesystem Presentation and cache stores create shard + // directories and commit durable files atomically. Keep this list explicit + // so the deny-by-default profile cannot silently turn configured writes into + // HTTP 500 responses. + for _, name := range []string{"fchmod", "fsync", "mkdirat", "renameat", "unlinkat"} { + if !allowed[name] { + t.Errorf("seccomp profile blocks required file-store syscall %q", name) + } + } +} + +func containsExact(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func inlinePythonLines(script string) []string { + // Join shell continuations so splitting a forbidden flag onto the next line + // cannot evade the contract check. + script = strings.ReplaceAll(script, "\\\r\n", " ") + script = strings.ReplaceAll(script, "\\\n", " ") + lines := strings.Split(script, "\n") + var violations []string + for index, line := range lines { + match := pythonInvocation.FindStringSubmatch(line) + if match == nil { + continue + } + arguments := strings.TrimSpace(match[len(match)-1]) + fields := strings.Fields(arguments) + if len(fields) == 0 { + continue + } + first := fields[0] + if first == "-" || strings.HasPrefix(first, "-c") || strings.Contains(arguments, "<<") { + violations = append(violations, fmt.Sprintf("%d: %s", index+1, strings.TrimSpace(line))) + } + } + return violations +} diff --git a/internal/iiif/presentation/v3/handler/handler.go b/internal/iiif/presentation/v3/handler/handler.go index 96353cb..da74be7 100644 --- a/internal/iiif/presentation/v3/handler/handler.go +++ b/internal/iiif/presentation/v3/handler/handler.go @@ -1,14 +1,20 @@ package handler import ( + "crypto/sha256" "crypto/subtle" "encoding/json" "errors" + "fmt" "io" "log/slog" + "mime" "net/http" "net/url" + "strconv" "strings" + "time" + "unicode/utf8" "github.com/libops/triplet/internal/cors" "github.com/libops/triplet/internal/iiif/presentation/v3/store" @@ -16,25 +22,34 @@ import ( "github.com/libops/triplet/internal/redact" ) -// Handler serves a minimal Presentation API v3 surface. +const ( + maxWriteBodyBytes = 8 << 20 + documentMediaType = `application/ld+json;profile="http://iiif.io/api/presentation/3/context.json"` +) + +// Handler serves byte-exact IIIF Presentation API v3 resources beneath a +// configured public prefix. type Handler struct { - prefix string - store store.Store - cors cors.Policy - writeEnabled bool - writeToken string - logger *slog.Logger + prefix string + publicBaseURL string + store store.Store + cors cors.Policy + writeEnabled bool + writeToken string + logger *slog.Logger } -// New constructs a presentation handler mounted at prefix. -func New(prefix string, st store.Store, corsPolicy cors.Policy, writeEnabled bool, writeToken string, logger *slog.Logger) *Handler { +// New constructs a Presentation handler mounted at prefix. publicBaseURL is +// authoritative when matching a resource's JSON-LD id to its request URL. +func New(prefix, publicBaseURL string, st store.Store, corsPolicy cors.Policy, writeEnabled bool, writeToken string, logger *slog.Logger) *Handler { return &Handler{ - prefix: strings.TrimRight(prefix, "/"), - store: st, - cors: corsPolicy, - writeEnabled: writeEnabled, - writeToken: writeToken, - logger: logger, + prefix: strings.TrimRight(prefix, "/"), + publicBaseURL: strings.TrimRight(publicBaseURL, "/"), + store: st, + cors: corsPolicy, + writeEnabled: writeEnabled, + writeToken: writeToken, + logger: logger, } } @@ -45,91 +60,26 @@ func (h *Handler) Register(mux *http.ServeMux) { // ServeHTTP implements http.Handler. func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet, http.MethodHead, http.MethodPut, http.MethodOptions: - default: - h.writeError(w, r, http.StatusMethodNotAllowed, "method not allowed") - return - } if r.Method == http.MethodOptions { - h.writeCORS(w, r) - methods := "GET, HEAD, OPTIONS" - if h.writeEnabled { - methods = "GET, HEAD, PUT, OPTIONS" - w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, If-Match") - } else { - w.Header().Set("Access-Control-Allow-Headers", "Content-Type") - } - w.Header().Set("Access-Control-Allow-Methods", methods) - w.WriteHeader(http.StatusNoContent) + h.writeOptions(w, r) return } - rest := strings.TrimPrefix(r.URL.Path, h.prefix) - rest = strings.Trim(rest, "/") - parts := strings.Split(rest, "/") - switch { - case len(parts) == 2 && parts[1] == "manifest": - h.serveManifest(w, r, parts[0]) - case len(parts) == 4 && parts[1] == "canvas" && parts[3] == "annotations": - h.serveAnnotationPage(w, r, parts[0], parts[2]) - default: - h.writeError(w, r, http.StatusNotFound, "not found") - } -} - -func (h *Handler) serveManifest(w http.ResponseWriter, r *http.Request, rawItemID string) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - h.writeError(w, r, http.StatusMethodNotAllowed, "method not allowed") - return - } - itemID, err := url.PathUnescape(rawItemID) - if err != nil || !validRequestID(itemID) { - h.writeError(w, r, http.StatusBadRequest, "invalid item id") + if r.Method != http.MethodGet && r.Method != http.MethodHead && r.Method != http.MethodPut && r.Method != http.MethodDelete { + h.writeMethodNotAllowed(w, r) return } - body, err := h.store.GetManifest(r.Context(), itemID) + resourceKey, publicID, err := h.requestResource(r) if err != nil { - if errors.Is(err, store.ErrNotFound) { - h.writeError(w, r, http.StatusNotFound, "manifest not found") - return - } - h.logger.Error("read manifest", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "err", err) - h.writeError(w, r, http.StatusInternalServerError, "failed to read manifest") - return - } - if err := validate.ValidateManifestBytes(body); err != nil { - h.logger.Error("validate manifest", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "err", err) - h.writeError(w, r, http.StatusInternalServerError, "invalid manifest") - return - } - h.writeDocumentHeaders(w, r) - w.Header().Set("ETag", store.DocumentETag(body)) - w.WriteHeader(http.StatusOK) - if r.Method == http.MethodHead { + h.writeError(w, r, http.StatusBadRequest, err.Error()) return } - if _, err := w.Write(body); err != nil { - h.logger.Warn("write manifest", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "err", err) - } -} -func (h *Handler) serveAnnotationPage(w http.ResponseWriter, r *http.Request, rawItemID, rawCanvasID string) { - itemID, err := url.PathUnescape(rawItemID) - if err != nil || !validRequestID(itemID) { - h.writeError(w, r, http.StatusBadRequest, "invalid item id") - return - } - canvasID, err := url.PathUnescape(rawCanvasID) - if err != nil || !validRequestID(canvasID) { - h.writeError(w, r, http.StatusBadRequest, "invalid canvas id") - return - } switch r.Method { case http.MethodGet, http.MethodHead: - h.getAnnotationPage(w, r, itemID, canvasID) - case http.MethodPut: + h.getResource(w, r, resourceKey, publicID) + case http.MethodPut, http.MethodDelete: if !h.writeEnabled { - h.writeError(w, r, http.StatusMethodNotAllowed, "method not allowed") + h.writeMethodNotAllowed(w, r) return } if !h.authorizedWrite(r) { @@ -137,71 +87,251 @@ func (h *Handler) serveAnnotationPage(w http.ResponseWriter, r *http.Request, ra h.writeError(w, r, http.StatusUnauthorized, "unauthorized") return } - h.putAnnotationPage(w, r, itemID, canvasID) - default: - h.writeError(w, r, http.StatusMethodNotAllowed, "method not allowed") + if r.Method == http.MethodPut { + h.putResource(w, r, resourceKey, publicID) + return + } + h.deleteResource(w, r, resourceKey) } } -func (h *Handler) getAnnotationPage(w http.ResponseWriter, r *http.Request, itemID, canvasID string) { - body, err := h.store.GetAnnotationPage(r.Context(), itemID, canvasID) +func (h *Handler) getResource(w http.ResponseWriter, r *http.Request, resourceKey, publicID string) { + document, err := h.store.Get(r.Context(), resourceKey) if err != nil { if errors.Is(err, store.ErrNotFound) { - h.writeError(w, r, http.StatusNotFound, "annotation page not found") + h.writeError(w, r, http.StatusNotFound, "presentation resource not found") return } - h.logger.Error("read annotation page", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "canvas_id", redact.Identifier(canvasID), "canvas_id_hash", redact.Hash(canvasID), "err", err) - h.writeError(w, r, http.StatusInternalServerError, "failed to read annotation page") + h.logStoreError("read presentation resource", resourceKey, err) + h.writeError(w, r, http.StatusInternalServerError, "failed to read presentation resource") return } - if err := validate.ValidateAnnotationPageBytes(body); err != nil { - h.logger.Error("validate annotation page", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "canvas_id", redact.Identifier(canvasID), "canvas_id_hash", redact.Hash(canvasID), "err", err) - h.writeError(w, r, http.StatusInternalServerError, "invalid annotation page") + resource, err := validate.ValidateResourceBytes(document.Body) + if err != nil || resource.ID != publicID { + validationErr := validationError(err, resource.ID, publicID) + h.logger.Error("validate stored presentation resource", "resource_key_hash", redact.Hash(resourceKey), "validation_error_hash", redact.Hash(validationErr.Error())) + h.writeError(w, r, http.StatusInternalServerError, "invalid stored presentation resource") return } + + etag := store.DocumentETag(document.Body) h.writeDocumentHeaders(w, r) - w.Header().Set("ETag", store.DocumentETag(body)) + w.Header().Set("ETag", etag) + if !document.ModifiedAt.IsZero() { + w.Header().Set("Last-Modified", document.ModifiedAt.UTC().Format(http.TimeFormat)) + } + if requestNotModified(r, etag, document.ModifiedAt) { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(document.Body))) w.WriteHeader(http.StatusOK) if r.Method == http.MethodHead { return } - if _, err := w.Write(body); err != nil { - h.logger.Warn("write annotation page", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "canvas_id", redact.Identifier(canvasID), "canvas_id_hash", redact.Hash(canvasID), "err", err) + if _, err := w.Write(document.Body); err != nil { + h.logger.Warn("write presentation resource", "resource_key_hash", redact.Hash(resourceKey), "resource_type", resource.Type, "err", err) } } -func (h *Handler) putAnnotationPage(w http.ResponseWriter, r *http.Request, itemID, canvasID string) { - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8<<20)) +func (h *Handler) putResource(w http.ResponseWriter, r *http.Request, resourceKey, publicID string) { + if !supportedContentType(r.Header.Get("Content-Type")) { + h.writeError(w, r, http.StatusUnsupportedMediaType, "Content-Type must be application/json or application/ld+json") + return + } + conditions, status, err := putPreconditions(r) if err != nil { + h.writeError(w, r, status, err.Error()) + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxWriteBodyBytes)) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + h.writeError(w, r, http.StatusRequestEntityTooLarge, "presentation resource exceeds the 8 MiB limit") + return + } h.writeError(w, r, http.StatusBadRequest, "invalid request body") return } - if err := validate.ValidateAnnotationPageBytes(body); err != nil { - h.writeError(w, r, http.StatusBadRequest, err.Error()) + resource, err := validate.ValidateResourceBytes(body) + if err != nil { + h.writeError(w, r, http.StatusBadRequest, "invalid presentation resource: "+err.Error()) return } - match := strings.TrimSpace(r.Header.Get("If-Match")) - if match == "" { - h.writeError(w, r, http.StatusPreconditionRequired, "If-Match is required") + if resource.ID != publicID { + h.writeError(w, r, http.StatusConflict, "presentation resource id must exactly match its public request URL") return } - if err := h.store.PutAnnotationPage(r.Context(), itemID, canvasID, body, match); err != nil { + created, err := h.store.Put(r.Context(), resourceKey, body, conditions) + if err != nil { if errors.Is(err, store.ErrPreconditionFailed) { - h.writeError(w, r, http.StatusPreconditionFailed, "annotation page precondition failed") + h.writeError(w, r, http.StatusPreconditionFailed, "presentation resource precondition failed") return } - h.logger.Error("write annotation page", "item_id", redact.Identifier(itemID), "item_id_hash", redact.Hash(itemID), "canvas_id", redact.Identifier(canvasID), "canvas_id_hash", redact.Hash(canvasID), "err", err) - h.writeError(w, r, http.StatusInternalServerError, "failed to store annotation page") + h.logStoreError("write presentation resource", resourceKey, err) + h.writeError(w, r, http.StatusInternalServerError, "failed to store presentation resource") return } h.writeCORS(w, r) w.Header().Set("ETag", store.DocumentETag(body)) + if created { + w.Header().Set("Location", publicID) + w.WriteHeader(http.StatusCreated) + return + } w.WriteHeader(http.StatusNoContent) } +func (h *Handler) deleteResource(w http.ResponseWriter, r *http.Request, resourceKey string) { + ifMatch := strings.TrimSpace(r.Header.Get("If-Match")) + if ifMatch == "" { + h.writeError(w, r, http.StatusPreconditionRequired, "If-Match is required") + return + } + if !validStrongIfMatch(ifMatch) { + h.writeError(w, r, http.StatusBadRequest, "If-Match must contain strong entity tags or *") + return + } + if err := h.store.Delete(r.Context(), resourceKey, ifMatch); err != nil { + if errors.Is(err, store.ErrPreconditionFailed) || errors.Is(err, store.ErrNotFound) { + h.writeError(w, r, http.StatusPreconditionFailed, "presentation resource precondition failed") + return + } + h.logStoreError("delete presentation resource", resourceKey, err) + h.writeError(w, r, http.StatusInternalServerError, "failed to delete presentation resource") + return + } + h.writeCORS(w, r) + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) requestResource(r *http.Request) (resourceKey, publicID string, err error) { + if r.URL.RawQuery != "" { + return "", "", errors.New("presentation resource URLs must not contain a query") + } + escapedPath := r.URL.EscapedPath() + escapedPrefix := (&url.URL{Path: h.prefix}).EscapedPath() + if !strings.HasPrefix(escapedPath, escapedPrefix+"/") { + return "", "", errors.New("invalid presentation resource path") + } + escapedRest := strings.TrimPrefix(escapedPath, escapedPrefix+"/") + decodedRest := strings.TrimPrefix(r.URL.Path, h.prefix+"/") + if escapedRest == "" || decodedRest == "" || strings.Contains(strings.ToLower(escapedRest), "%2f") || strings.Contains(strings.ToLower(escapedRest), "%5c") { + return "", "", errors.New("invalid presentation resource path") + } + segments := strings.Split(decodedRest, "/") + canonical := make([]string, 0, len(segments)) + for _, segment := range segments { + if segment == "" || segment == "." || segment == ".." || !utf8.ValidString(segment) || strings.ContainsAny(segment, "\\\x00\n\r") { + return "", "", errors.New("invalid presentation resource path") + } + canonical = append(canonical, url.PathEscape(segment)) + } + resourceKey = strings.Join(canonical, "/") + if resourceKey != escapedRest || len(resourceKey) > store.MaxResourceKeyBytes { + return "", "", errors.New("presentation resource path is not canonically escaped or is too long") + } + return resourceKey, h.publicBaseURL + h.prefix + "/" + resourceKey, nil +} + +func putPreconditions(r *http.Request) (store.Preconditions, int, error) { + ifMatch := strings.TrimSpace(r.Header.Get("If-Match")) + ifNoneMatch := strings.TrimSpace(r.Header.Get("If-None-Match")) + if ifMatch == "" && ifNoneMatch == "" { + return store.Preconditions{}, http.StatusPreconditionRequired, errors.New("If-None-Match: * is required to create, or If-Match is required to replace") + } + if ifMatch != "" && ifNoneMatch != "" { + return store.Preconditions{}, http.StatusBadRequest, errors.New("If-Match and If-None-Match cannot be combined") + } + if ifNoneMatch != "" && ifNoneMatch != "*" { + return store.Preconditions{}, http.StatusBadRequest, errors.New("If-None-Match must be * for a conditional create") + } + if ifMatch != "" && !validStrongIfMatch(ifMatch) { + return store.Preconditions{}, http.StatusBadRequest, errors.New("If-Match must contain strong entity tags or *") + } + return store.Preconditions{IfMatch: ifMatch, IfNoneMatch: ifNoneMatch}, 0, nil +} + +func validStrongIfMatch(value string) bool { + value = strings.TrimSpace(value) + if value == "*" { + return true + } + if value == "" { + return false + } + for _, candidate := range strings.Split(value, ",") { + candidate = strings.TrimSpace(candidate) + if len(candidate) < 2 || candidate[0] != '"' || candidate[len(candidate)-1] != '"' || strings.HasPrefix(candidate, "W/") || strings.ContainsAny(candidate[1:len(candidate)-1], "\"\r\n") { + return false + } + } + return true +} + +func supportedContentType(value string) bool { + mediaType, _, err := mime.ParseMediaType(value) + if err != nil { + return false + } + return mediaType == "application/json" || mediaType == "application/ld+json" +} + +func requestNotModified(r *http.Request, etag string, modifiedAt time.Time) bool { + ifNoneMatch := strings.TrimSpace(r.Header.Get("If-None-Match")) + if ifNoneMatch != "" { + for _, candidate := range strings.Split(ifNoneMatch, ",") { + candidate = strings.TrimSpace(candidate) + if candidate == "*" || strings.TrimPrefix(candidate, "W/") == etag { + return true + } + } + return false + } + if modifiedAt.IsZero() { + return false + } + ifModifiedSince, err := http.ParseTime(r.Header.Get("If-Modified-Since")) + if err != nil { + return false + } + return !modifiedAt.UTC().Truncate(time.Second).After(ifModifiedSince.UTC()) +} + +func validationError(err error, actualID, expectedID string) error { + if err != nil { + return err + } + return fmt.Errorf("resource id hash %s does not match request id hash %s", redact.Hash(actualID), redact.Hash(expectedID)) +} + +func (h *Handler) writeOptions(w http.ResponseWriter, r *http.Request) { + h.writeCORS(w, r) + methods := "GET, HEAD, OPTIONS" + headers := "Content-Type, If-None-Match" + if h.writeEnabled { + methods = "GET, HEAD, PUT, DELETE, OPTIONS" + headers = "Authorization, Content-Type, If-Match, If-None-Match" + } + w.Header().Set("Access-Control-Allow-Headers", headers) + w.Header().Set("Access-Control-Allow-Methods", methods) + w.Header().Set("Allow", methods) + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) writeMethodNotAllowed(w http.ResponseWriter, r *http.Request) { + methods := "GET, HEAD, OPTIONS" + if h.writeEnabled { + methods = "GET, HEAD, PUT, DELETE, OPTIONS" + } + w.Header().Set("Allow", methods) + h.writeError(w, r, http.StatusMethodNotAllowed, "method not allowed") +} + func (h *Handler) writeDocumentHeaders(w http.ResponseWriter, r *http.Request) { h.writeCORS(w, r) - w.Header().Set("Content-Type", `application/ld+json;profile="http://iiif.io/api/presentation/3/context.json"`) + w.Header().Set("Content-Type", documentMediaType) } func (h *Handler) writeCORS(w http.ResponseWriter, r *http.Request) { @@ -213,7 +343,9 @@ func (h *Handler) authorizedWrite(r *http.Request) bool { if token == "" || h.writeToken == "" { return false } - return subtle.ConstantTimeCompare([]byte(token), []byte(h.writeToken)) == 1 + candidateDigest := sha256.Sum256([]byte(token)) + expectedDigest := sha256.Sum256([]byte(h.writeToken)) + return subtle.ConstantTimeCompare(candidateDigest[:], expectedDigest[:]) == 1 } func bearerToken(r *http.Request) string { @@ -224,15 +356,18 @@ func bearerToken(r *http.Request) string { return strings.TrimSpace(auth[len("Bearer "):]) } -func validRequestID(id string) bool { - return id != "" && len(id) <= 255 && !strings.ContainsAny(id, "\x00\n\r") +func (h *Handler) logStoreError(message, resourceKey string, err error) { + h.logger.Error(message, "resource_key_hash", redact.Hash(resourceKey), "err", err) } -func (h *Handler) writeError(w http.ResponseWriter, r *http.Request, status int, msg string) { +func (h *Handler) writeError(w http.ResponseWriter, r *http.Request, status int, message string) { h.writeCORS(w, r) w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(errorResponse{Error: msg}) + if r.Method == http.MethodHead { + return + } + _ = json.NewEncoder(w).Encode(errorResponse{Error: message}) } type errorResponse struct { diff --git a/internal/iiif/presentation/v3/handler/handler_test.go b/internal/iiif/presentation/v3/handler/handler_test.go index e1ecaa5..4667df3 100644 --- a/internal/iiif/presentation/v3/handler/handler_test.go +++ b/internal/iiif/presentation/v3/handler/handler_test.go @@ -1,13 +1,13 @@ package handler import ( - "encoding/json" + "bytes" + "fmt" "io" "log/slog" "net/http" "net/http/httptest" - "os" - "path/filepath" + "strconv" "strings" "testing" @@ -15,355 +15,508 @@ import ( pstore "github.com/libops/triplet/internal/iiif/presentation/v3/store" ) -func setupTestServer(t *testing.T) *httptest.Server { - t.Helper() - return setupTestServerWithWrites(t, false, "") -} +const ( + testPrefix = "/presentation/v3" + testPublicBase = "https://iiif.example.org" + testWriteToken = "test-token" +) -func setupTestServerWithWrites(t *testing.T, writeEnabled bool, writeToken string) *httptest.Server { - t.Helper() - return setupTestServerWithWritesAndCORS(t, writeEnabled, writeToken, nil) +type presentationTestServer struct { + server *httptest.Server + store *pstore.FileStore } -func setupTestServerWithWritesAndCORS(t *testing.T, writeEnabled bool, writeToken string, allowedOrigins []string) *httptest.Server { +func newPresentationTestServer(t *testing.T, writeEnabled bool, allowedOrigins []string, seed map[string][]byte) presentationTestServer { t.Helper() - root := t.TempDir() - itemDir := filepath.Join(root, "item-1") - if err := os.MkdirAll(itemDir, 0o755); err != nil { - t.Fatal(err) - } - body := `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/presentation/v3/item-1/manifest","type":"Manifest","label":{"en":["Item 1"]},"items":[{"id":"http://example.test/presentation/v3/item-1/canvas/1","type":"Canvas"}]}` - if err := os.WriteFile(filepath.Join(itemDir, "manifest.json"), []byte(body), 0o600); err != nil { - t.Fatal(err) - } - annoDir := filepath.Join(itemDir, "canvas", "canvas-1") - if err := os.MkdirAll(annoDir, 0o755); err != nil { - t.Fatal(err) - } - anno := `{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/presentation/v3/item-1/canvas/canvas-1/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/annotations/1","type":"Annotation","textGranularity":"line","motivation":["supplementing"],"body":{"type":"TextualBody","value":"hello"},"target":{"type":"SpecificResource","source":"http://example.test/presentation/v3/item-1/canvas/canvas-1","selector":{"type":"FragmentSelector","value":"xywh=1,2,3,4"}}}]}` - if err := os.WriteFile(filepath.Join(annoDir, "annotations.json"), []byte(anno), 0o600); err != nil { - t.Fatal(err) - } - st, err := pstore.NewFileStore(root) + st, err := pstore.NewFileStore(t.TempDir()) if err != nil { t.Fatal(err) } + for key, body := range seed { + if _, err := st.Put(t.Context(), key, body, pstore.Preconditions{IfNoneMatch: "*"}); err != nil { + t.Fatalf("seed %q: %v", key, err) + } + } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - h := New("/presentation/v3", st, cors.New(allowedOrigins, "ETag"), writeEnabled, writeToken, logger) + h := New( + testPrefix, + testPublicBase, + st, + cors.New(allowedOrigins, "ETag, Last-Modified, Content-Length, Location"), + writeEnabled, + testWriteToken, + logger, + ) mux := http.NewServeMux() h.Register(mux) - return httptest.NewServer(mux) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return presentationTestServer{server: srv, store: st} } -func TestManifest(t *testing.T) { - srv := setupTestServer(t) - defer srv.Close() - resp, err := http.Get(srv.URL + "/presentation/v3/item-1/manifest") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d", resp.StatusCode) - } - if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "" { - t.Fatalf("CORS = %q", got) - } - if got := resp.Header.Get("Content-Type"); got == "" { - t.Fatal("missing content-type") - } +func publicID(key string) string { + return testPublicBase + testPrefix + "/" + key } -func TestManifestPutMethodNotAllowed(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/manifest", strings.NewReader(`{}`)) +func manifestBody(key string) []byte { + return []byte(fmt.Sprintf(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":%q,"type":"Manifest","label":{"en":["Manifest"]},"items":[]}`, publicID(key))) +} + +func canvasBody(key string) []byte { + return []byte(fmt.Sprintf(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":%q,"type":"Canvas","width":100,"height":200,"items":[]}`, publicID(key))) +} + +func annotationPageBody(key, value string) []byte { + return []byte(fmt.Sprintf(`{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":%q,"type":"AnnotationPage","items":[{"id":%q,"type":"Annotation","motivation":"supplementing","body":{"type":"TextualBody","value":%q},"target":"https://iiif.example.org/canvas/1#xywh=1,2,3,4","textGranularity":"line"}]}`, publicID(key), publicID(key+"/items/1"), value)) +} + +func annotationBody(key string) []byte { + return []byte(fmt.Sprintf(`{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":%q,"type":"Annotation","target":"https://iiif.example.org/canvas/1","textGranularity":"token"}`, publicID(key))) +} + +func collectionBody(key string) []byte { + return []byte(fmt.Sprintf(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":%q,"type":"Collection","label":{"en":["Collection"]},"items":[]}`, publicID(key))) +} + +func annotationCollectionBody(key string) []byte { + return []byte(fmt.Sprintf(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":%q,"type":"AnnotationCollection"}`, publicID(key))) +} + +func authorizedPUT(t *testing.T, endpoint string, body []byte, ifMatch, ifNoneMatch string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(body)) if err != nil { t.Fatal(err) } - req.Header.Set("Authorization", "Bearer test-token") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) + req.Header.Set("Authorization", "Bearer "+testWriteToken) + req.Header.Set("Content-Type", "application/ld+json") + if ifMatch != "" { + req.Header.Set("If-Match", ifMatch) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusMethodNotAllowed { - t.Fatalf("status = %d", resp.StatusCode) + if ifNoneMatch != "" { + req.Header.Set("If-None-Match", ifNoneMatch) } -} - -func TestAnnotationPageGet(t *testing.T) { - srv := setupTestServer(t) - defer srv.Close() - resp, err := http.Get(srv.URL + "/presentation/v3/item-1/canvas/canvas-1/annotations") + resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d", resp.StatusCode) - } + return resp } -func TestAnnotationPageHead(t *testing.T) { - srv := setupTestServer(t) - defer srv.Close() - req, err := http.NewRequest(http.MethodHead, srv.URL+"/presentation/v3/item-1/canvas/canvas-1/annotations", nil) +func TestResourceGetHeadAndConditionalRequests(t *testing.T) { + key := "items/item-1/manifest" + body := manifestBody(key) + testServer := newPresentationTestServer(t, false, []string{"https://viewer.example.org"}, map[string][]byte{key: body}) + endpoint := testServer.server.URL + testPrefix + "/" + key + + req, err := http.NewRequest(http.MethodGet, endpoint, nil) if err != nil { t.Fatal(err) } + req.Header.Set("Origin", "https://viewer.example.org") resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d", resp.StatusCode) - } - b, err := io.ReadAll(resp.Body) + got, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() if err != nil { t.Fatal(err) } - if len(b) != 0 { - t.Fatalf("expected empty body, got %q", string(b)) + if resp.StatusCode != http.StatusOK || !bytes.Equal(got, body) { + t.Fatalf("GET status/body = %d, %q", resp.StatusCode, got) } -} - -func TestAnnotationPageCORSExposesETag(t *testing.T) { - srv := setupTestServerWithWritesAndCORS(t, true, "test-token", []string{"https://editor.example.edu"}) - defer srv.Close() - req, err := http.NewRequest(http.MethodGet, srv.URL+"/presentation/v3/item-1/canvas/canvas-1/annotations", nil) - if err != nil { - t.Fatal(err) + if got := resp.Header.Get("Content-Type"); got != documentMediaType { + t.Fatalf("Content-Type = %q", got) } - req.Header.Set("Origin", "https://editor.example.edu") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) + if got := resp.Header.Get("Content-Length"); got != strconv.Itoa(len(body)) { + t.Fatalf("Content-Length = %q", got) } - defer resp.Body.Close() - if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "https://editor.example.edu" { + etag := resp.Header.Get("ETag") + lastModified := resp.Header.Get("Last-Modified") + if etag == "" || lastModified == "" { + t.Fatalf("missing validators: ETag=%q Last-Modified=%q", etag, lastModified) + } + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "https://viewer.example.org" { t.Fatalf("Access-Control-Allow-Origin = %q", got) } - if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "ETag" { + if got := resp.Header.Get("Access-Control-Expose-Headers"); got != "ETag, Last-Modified, Content-Length, Location" { t.Fatalf("Access-Control-Expose-Headers = %q", got) } - if got := resp.Header.Get("ETag"); got == "" { - t.Fatal("missing ETag") - } -} -func TestAnnotationPagePut(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - body := `{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/presentation/v3/item-1/canvas/canvas-2/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/annotations/2","type":"Annotation","textGranularity":"line","motivation":["supplementing"],"body":{"type":"TextualBody","value":"world"},"target":{"type":"SpecificResource","source":"http://example.test/presentation/v3/item-1/canvas/canvas-2","selector":{"type":"FragmentSelector","value":"xywh=5,6,7,8"}}}]}` - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/canvas/canvas-2/annotations", strings.NewReader(body)) + headReq, _ := http.NewRequest(http.MethodHead, endpoint, nil) + headResp, err := http.DefaultClient.Do(headReq) if err != nil { t.Fatal(err) } - req.Header.Set("Content-Type", "application/ld+json") - req.Header.Set("Authorization", "Bearer test-token") - req.Header.Set("If-Match", "*") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - t.Fatalf("status = %d", resp.StatusCode) + headBody, _ := io.ReadAll(headResp.Body) + _ = headResp.Body.Close() + if headResp.StatusCode != http.StatusOK || len(headBody) != 0 || headResp.Header.Get("Content-Length") != strconv.Itoa(len(body)) { + t.Fatalf("HEAD status/body/length = %d, %q, %q", headResp.StatusCode, headBody, headResp.Header.Get("Content-Length")) } - getResp, err := http.Get(srv.URL + "/presentation/v3/item-1/canvas/canvas-2/annotations") + for name, header := range map[string]string{"strong ETag": etag, "weak ETag": "W/" + etag} { + t.Run(name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, endpoint, nil) + req.Header.Set("If-None-Match", header) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotModified { + t.Fatalf("status = %d", resp.StatusCode) + } + }) + } + modifiedReq, _ := http.NewRequest(http.MethodGet, endpoint, nil) + modifiedReq.Header.Set("If-Modified-Since", lastModified) + modifiedResp, err := http.DefaultClient.Do(modifiedReq) if err != nil { t.Fatal(err) } - defer getResp.Body.Close() - if getResp.StatusCode != http.StatusOK { - t.Fatalf("get status = %d", getResp.StatusCode) + _ = modifiedResp.Body.Close() + if modifiedResp.StatusCode != http.StatusNotModified { + t.Fatalf("If-Modified-Since status = %d", modifiedResp.StatusCode) } - if getResp.Header.Get("ETag") == "" { - t.Fatal("GET response missing ETag") +} + +func TestConditionalPutSupportsGenericResourceTypes(t *testing.T) { + testServer := newPresentationTestServer(t, true, nil, nil) + tests := []struct { + name string + key string + body []byte + }{ + {name: "manifest", key: "items/1/manifest"}, + {name: "canvas", key: "items/1/canvas/1"}, + {name: "annotation page", key: "items/1/canvas/1/annotations"}, + {name: "annotation", key: "items/1/canvas/1/annotations/items/1"}, + {name: "collection", key: "collections/1"}, + {name: "annotation collection", key: "annotation-collections/1"}, + } + tests[0].body = manifestBody(tests[0].key) + tests[1].body = canvasBody(tests[1].key) + tests[2].body = annotationPageBody(tests[2].key, "hello") + tests[3].body = annotationBody(tests[3].key) + tests[4].body = collectionBody(tests[4].key) + tests[5].body = annotationCollectionBody(tests[5].key) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + endpoint := testServer.server.URL + testPrefix + "/" + test.key + resp := authorizedPUT(t, endpoint, test.body, "", "*") + _ = resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + t.Fatalf("PUT status = %d", resp.StatusCode) + } + if got := resp.Header.Get("Location"); got != publicID(test.key) { + t.Fatalf("Location = %q", got) + } + getResp, err := http.Get(endpoint) + if err != nil { + t.Fatal(err) + } + got, _ := io.ReadAll(getResp.Body) + _ = getResp.Body.Close() + if getResp.StatusCode != http.StatusOK || !bytes.Equal(got, test.body) { + t.Fatalf("GET status/body = %d, %q", getResp.StatusCode, got) + } + }) } } -func TestAnnotationPagePutRequiresIfMatch(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - body := `{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/presentation/v3/item-1/canvas/canvas-2/annotations","type":"AnnotationPage","items":[]}` - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/canvas/canvas-2/annotations", strings.NewReader(body)) +func TestReplaceAndDeleteUseStrongPreconditions(t *testing.T) { + key := "items/1/canvas/1/annotations" + first := annotationPageBody(key, "first") + testServer := newPresentationTestServer(t, true, nil, map[string][]byte{key: first}) + endpoint := testServer.server.URL + testPrefix + "/" + key + getResp, err := http.Get(endpoint) if err != nil { t.Fatal(err) } - req.Header.Set("Authorization", "Bearer test-token") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) + firstETag := getResp.Header.Get("ETag") + _ = getResp.Body.Close() + + second := annotationPageBody(key, "second") + updateResp := authorizedPUT(t, endpoint, second, firstETag, "") + _ = updateResp.Body.Close() + if updateResp.StatusCode != http.StatusNoContent { + t.Fatalf("update status = %d", updateResp.StatusCode) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusPreconditionRequired { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusPreconditionRequired) + secondETag := updateResp.Header.Get("ETag") + if secondETag == "" || secondETag == firstETag { + t.Fatalf("updated ETag = %q", secondETag) + } + staleResp := authorizedPUT(t, endpoint, first, firstETag, "") + _ = staleResp.Body.Close() + if staleResp.StatusCode != http.StatusPreconditionFailed { + t.Fatalf("stale PUT status = %d", staleResp.StatusCode) } -} -func TestAnnotationPagePutHonorsIfMatch(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - url := srv.URL + "/presentation/v3/item-1/canvas/canvas-1/annotations" - getResp, err := http.Get(url) + deleteReq, _ := http.NewRequest(http.MethodDelete, endpoint, nil) + deleteReq.Header.Set("Authorization", "Bearer "+testWriteToken) + deleteReq.Header.Set("If-Match", firstETag) + deleteResp, err := http.DefaultClient.Do(deleteReq) if err != nil { t.Fatal(err) } - etag := getResp.Header.Get("ETag") - _ = getResp.Body.Close() - if etag == "" { - t.Fatal("GET response missing ETag") + _ = deleteResp.Body.Close() + if deleteResp.StatusCode != http.StatusPreconditionFailed { + t.Fatalf("stale DELETE status = %d", deleteResp.StatusCode) } - body := `{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/presentation/v3/item-1/canvas/canvas-1/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/annotations/3","type":"Annotation","textGranularity":"line","motivation":["supplementing"],"body":{"type":"TextualBody","value":"updated"},"target":{"type":"SpecificResource","source":"http://example.test/presentation/v3/item-1/canvas/canvas-1","selector":{"type":"FragmentSelector","value":"xywh=5,6,7,8"}}}]}` - req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(body)) + deleteReq, _ = http.NewRequest(http.MethodDelete, endpoint, nil) + deleteReq.Header.Set("Authorization", "Bearer "+testWriteToken) + deleteReq.Header.Set("If-Match", secondETag) + deleteResp, err = http.DefaultClient.Do(deleteReq) if err != nil { t.Fatal(err) } - req.Header.Set("Content-Type", "application/ld+json") - req.Header.Set("Authorization", "Bearer test-token") - req.Header.Set("If-Match", etag) - resp, err := http.DefaultClient.Do(req) + _ = deleteResp.Body.Close() + if deleteResp.StatusCode != http.StatusNoContent { + t.Fatalf("matched DELETE status = %d", deleteResp.StatusCode) + } + missingResp, err := http.Get(endpoint) if err != nil { t.Fatal(err) } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - t.Fatalf("matched PUT status = %d", resp.StatusCode) + _ = missingResp.Body.Close() + if missingResp.StatusCode != http.StatusNotFound { + t.Fatalf("deleted GET status = %d", missingResp.StatusCode) } +} - req, err = http.NewRequest(http.MethodPut, url, strings.NewReader(body)) - if err != nil { - t.Fatal(err) +func TestPutPreconditionContract(t *testing.T) { + existingKey := "items/1/manifest" + existingBody := manifestBody(existingKey) + testServer := newPresentationTestServer(t, true, nil, map[string][]byte{existingKey: existingBody}) + tests := []struct { + name string + key string + ifMatch string + ifNoneMatch string + want int + }{ + {name: "condition required", key: "items/2/manifest", want: http.StatusPreconditionRequired}, + {name: "create existing", key: existingKey, ifNoneMatch: "*", want: http.StatusPreconditionFailed}, + {name: "replace missing wildcard", key: "items/2/manifest", ifMatch: "*", want: http.StatusPreconditionFailed}, + {name: "both conditions", key: "items/2/manifest", ifMatch: "*", ifNoneMatch: "*", want: http.StatusBadRequest}, + {name: "non wildcard If-None-Match", key: "items/2/manifest", ifNoneMatch: `"etag"`, want: http.StatusBadRequest}, + {name: "weak If-Match", key: existingKey, ifMatch: `W/"etag"`, want: http.StatusBadRequest}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resp := authorizedPUT(t, testServer.server.URL+testPrefix+"/"+test.key, manifestBody(test.key), test.ifMatch, test.ifNoneMatch) + _ = resp.Body.Close() + if resp.StatusCode != test.want { + t.Fatalf("status = %d, want %d", resp.StatusCode, test.want) + } + }) } - req.Header.Set("Content-Type", "application/ld+json") - req.Header.Set("Authorization", "Bearer test-token") - req.Header.Set("If-Match", etag) - resp, err = http.DefaultClient.Do(req) +} + +func TestWriteAuthenticationMediaTypeValidationAndLimit(t *testing.T) { + testServer := newPresentationTestServer(t, true, nil, nil) + key := "items/1/manifest" + endpoint := testServer.server.URL + testPrefix + "/" + key + body := manifestBody(key) + + unauthorized, _ := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(body)) + unauthorized.Header.Set("Content-Type", "application/json") + unauthorized.Header.Set("If-None-Match", "*") + resp, err := http.DefaultClient.Do(unauthorized) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusPreconditionFailed { - t.Fatalf("stale PUT status = %d, want %d", resp.StatusCode, http.StatusPreconditionFailed) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized || resp.Header.Get("WWW-Authenticate") == "" { + t.Fatalf("unauthorized response = %d, %q", resp.StatusCode, resp.Header.Get("WWW-Authenticate")) } -} -func TestAnnotationPagePutDisabled(t *testing.T) { - srv := setupTestServer(t) - defer srv.Close() - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/canvas/canvas-2/annotations", strings.NewReader(`{}`)) + badMedia, _ := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(body)) + badMedia.Header.Set("Authorization", "Bearer "+testWriteToken) + badMedia.Header.Set("Content-Type", "text/plain") + badMedia.Header.Set("If-None-Match", "*") + resp, err = http.DefaultClient.Do(badMedia) if err != nil { t.Fatal(err) } - resp, err := http.DefaultClient.Do(req) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("bad media status = %d", resp.StatusCode) + } + + tooLarge, _ := http.NewRequest(http.MethodPut, endpoint, strings.NewReader(strings.Repeat("x", maxWriteBodyBytes+1))) + tooLarge.Header.Set("Authorization", "Bearer "+testWriteToken) + tooLarge.Header.Set("Content-Type", "application/json") + tooLarge.Header.Set("If-None-Match", "*") + resp, err = http.DefaultClient.Do(tooLarge) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusMethodNotAllowed { - t.Fatalf("status = %d", resp.StatusCode) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("large body status = %d", resp.StatusCode) } } -func TestAnnotationPagePutRequiresBearerToken(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/canvas/canvas-2/annotations", strings.NewReader(`{}`)) - if err != nil { - t.Fatal(err) +func TestPutRejectsInvalidResourceAndIDMismatch(t *testing.T) { + testServer := newPresentationTestServer(t, true, nil, nil) + key := "items/1/manifest" + endpoint := testServer.server.URL + testPrefix + "/" + key + tests := []struct { + name string + body []byte + want int + }{ + {name: "malformed", body: []byte(`{`), want: http.StatusBadRequest}, + {name: "unsupported type", body: []byte(fmt.Sprintf(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":%q,"type":"Sequence","items":[]}`, publicID(key))), want: http.StatusBadRequest}, + {name: "ID mismatch", body: manifestBody("other/manifest"), want: http.StatusConflict}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resp := authorizedPUT(t, endpoint, test.body, "", "*") + _ = resp.Body.Close() + if resp.StatusCode != test.want { + t.Fatalf("status = %d, want %d", resp.StatusCode, test.want) + } + }) } +} + +func TestDeleteContract(t *testing.T) { + key := "items/1/manifest" + testServer := newPresentationTestServer(t, true, nil, map[string][]byte{key: manifestBody(key)}) + endpoint := testServer.server.URL + testPrefix + "/" + key + + req, _ := http.NewRequest(http.MethodDelete, endpoint, nil) + req.Header.Set("Authorization", "Bearer "+testWriteToken) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusUnauthorized { - t.Fatalf("status = %d", resp.StatusCode) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusPreconditionRequired { + t.Fatalf("missing If-Match status = %d", resp.StatusCode) } -} -func TestAnnotationPagePutValidationError(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/canvas/canvas-2/annotations", strings.NewReader(`{"type":"AnnotationPage"}`)) + req, _ = http.NewRequest(http.MethodDelete, endpoint, nil) + req.Header.Set("Authorization", "Bearer "+testWriteToken) + req.Header.Set("If-Match", "*") + resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } - req.Header.Set("Authorization", "Bearer test-token") - resp, err := http.DefaultClient.Do(req) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("wildcard delete status = %d", resp.StatusCode) + } + + req, _ = http.NewRequest(http.MethodDelete, endpoint, nil) + req.Header.Set("Authorization", "Bearer "+testWriteToken) + req.Header.Set("If-Match", "*") + resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("status = %d", resp.StatusCode) - } - var got map[string]string - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { - t.Fatalf("decode error response: %v", err) - } - if got["error"] == "" { - t.Fatal("missing error message") + _ = resp.Body.Close() + if resp.StatusCode != http.StatusPreconditionFailed { + t.Fatalf("missing delete status = %d", resp.StatusCode) } } -func TestRejectsOverlongIDs(t *testing.T) { - srv := setupTestServerWithWrites(t, true, "test-token") - defer srv.Close() - tooLong := strings.Repeat("a", 256) - - resp, err := http.Get(srv.URL + "/presentation/v3/" + tooLong + "/manifest") +func TestCORSOptionsAndMethodAllow(t *testing.T) { + testServer := newPresentationTestServer(t, true, []string{"https://editor.example.org"}, nil) + endpoint := testServer.server.URL + testPrefix + "/items/1/manifest" + req, _ := http.NewRequest(http.MethodOptions, endpoint, nil) + req.Header.Set("Origin", "https://editor.example.org") + resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("manifest status = %d", resp.StatusCode) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("OPTIONS status = %d", resp.StatusCode) } - - body := `{"@context":["http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/annotations","type":"AnnotationPage","items":[]}` - req, err := http.NewRequest(http.MethodPut, srv.URL+"/presentation/v3/item-1/canvas/"+tooLong+"/annotations", strings.NewReader(body)) - if err != nil { - t.Fatal(err) + if got := resp.Header.Get("Access-Control-Allow-Methods"); got != "GET, HEAD, PUT, DELETE, OPTIONS" { + t.Fatalf("Access-Control-Allow-Methods = %q", got) } - req.Header.Set("Authorization", "Bearer test-token") - resp, err = http.DefaultClient.Do(req) + if got := resp.Header.Get("Access-Control-Allow-Headers"); got != "Authorization, Content-Type, If-Match, If-None-Match" { + t.Fatalf("Access-Control-Allow-Headers = %q", got) + } + + patchReq, _ := http.NewRequest(http.MethodPatch, endpoint, nil) + patchResp, err := http.DefaultClient.Do(patchReq) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("annotation status = %d", resp.StatusCode) + _ = patchResp.Body.Close() + if patchResp.StatusCode != http.StatusMethodNotAllowed || !strings.Contains(patchResp.Header.Get("Allow"), "DELETE") { + t.Fatalf("PATCH response = %d, Allow %q", patchResp.StatusCode, patchResp.Header.Get("Allow")) } } -func TestManifestNotFound(t *testing.T) { - srv := setupTestServer(t) - defer srv.Close() - resp, err := http.Get(srv.URL + "/presentation/v3/missing/manifest") - if err != nil { - t.Fatal(err) +func TestRejectsNonCanonicalAndQueryResourcePaths(t *testing.T) { + testServer := newPresentationTestServer(t, false, nil, nil) + for _, rawURL := range []string{ + testServer.server.URL + testPrefix + "/items/%69tem/manifest", + testServer.server.URL + testPrefix + "/items/item%2Fmanifest", + testServer.server.URL + testPrefix + "/items/item/manifest?view=1", + } { + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("GET %q status = %d", rawURL, resp.StatusCode) + } } - defer resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d", resp.StatusCode) +} + +func TestInvalidStoredResourcesFailClosed(t *testing.T) { + tests := []struct { + name string + body []byte + }{ + {name: "invalid JSON", body: []byte(`{`)}, + {name: "mismatched ID", body: manifestBody("other/manifest")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + key := "items/1/manifest" + testServer := newPresentationTestServer(t, false, nil, map[string][]byte{key: test.body}) + resp, err := http.Get(testServer.server.URL + testPrefix + "/" + key) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("status = %d", resp.StatusCode) + } + }) } } -func TestAnnotationPageNotFound(t *testing.T) { - srv := setupTestServer(t) - defer srv.Close() - resp, err := http.Get(srv.URL + "/presentation/v3/item-1/canvas/missing/annotations") +func TestNotFoundAndWritesDisabled(t *testing.T) { + testServer := newPresentationTestServer(t, false, nil, nil) + endpoint := testServer.server.URL + testPrefix + "/items/1/manifest" + resp, err := http.Get(endpoint) if err != nil { t.Fatal(err) } - defer resp.Body.Close() + _ = resp.Body.Close() if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d", resp.StatusCode) + t.Fatalf("GET status = %d", resp.StatusCode) + } + resp = authorizedPUT(t, endpoint, manifestBody("items/1/manifest"), "", "*") + _ = resp.Body.Close() + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("disabled PUT status = %d", resp.StatusCode) } } diff --git a/internal/iiif/presentation/v3/store/file.go b/internal/iiif/presentation/v3/store/file.go index 10775a3..3df102e 100644 --- a/internal/iiif/presentation/v3/store/file.go +++ b/internal/iiif/presentation/v3/store/file.go @@ -2,6 +2,8 @@ package store import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io/fs" @@ -11,8 +13,9 @@ import ( "sync" ) -// FileStore reads manifest documents from a filesystem root. -// Manifests are stored at {root}/{itemID}/manifest.json. +// FileStore persists path-keyed Presentation resources beneath a filesystem +// root. Keys are hashed before becoming filenames, avoiding path traversal, +// platform filename limits, and resource/directory name collisions. type FileStore struct { root string realRoot string @@ -39,85 +42,114 @@ func NewFileStore(root string) (*FileStore, error) { return &FileStore{root: abs, realRoot: realRoot}, nil } -// GetManifest implements Store. -func (s *FileStore) GetManifest(_ context.Context, itemID string) ([]byte, error) { - path, err := s.resolveRead(itemID, "manifest.json") +// Get implements Store. +func (s *FileStore) Get(_ context.Context, resourceKey string) (Document, error) { + path, err := s.resourcePath(resourceKey) if err != nil { - return nil, err + return Document{}, err } - b, err := os.ReadFile(path) + s.mu.Lock() + defer s.mu.Unlock() + path, err = s.resolveRead(path) if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil, ErrNotFound - } - return nil, err + return Document{}, err } - return b, nil -} - -// GetAnnotationPage implements Store. -func (s *FileStore) GetAnnotationPage(_ context.Context, itemID, canvasID string) ([]byte, error) { - path, err := s.resolveRead(itemID, "canvas", canvasID, "annotations.json") + body, err := os.ReadFile(path) if err != nil { - return nil, err + if errors.Is(err, fs.ErrNotExist) { + return Document{}, ErrNotFound + } + return Document{}, err } - b, err := os.ReadFile(path) + info, err := os.Stat(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { - return nil, ErrNotFound + return Document{}, ErrNotFound } - return nil, err + return Document{}, err } - return b, nil + return Document{Body: body, ModifiedAt: info.ModTime().UTC()}, nil } -// PutAnnotationPage implements Store. -func (s *FileStore) PutAnnotationPage(_ context.Context, itemID, canvasID string, body []byte, ifMatch string) error { - path, err := s.resolve(itemID, "canvas", canvasID, "annotations.json") +// Put implements Store. +func (s *FileStore) Put(_ context.Context, resourceKey string, body []byte, conditions Preconditions) (bool, error) { + path, err := s.resourcePath(resourceKey) if err != nil { - return err + return false, err } s.mu.Lock() defer s.mu.Unlock() dir := filepath.Dir(path) if err := s.ensureCreatePathContained(dir); err != nil { - return err + return false, err } if err := os.MkdirAll(dir, 0o750); err != nil { - return err + return false, err } if err := s.ensureContained(dir); err != nil { - return err + return false, err } - if err := s.checkAnnotationPagePrecondition(path, ifMatch); err != nil { - return err + current, err := os.ReadFile(path) + exists := err == nil + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return false, err } - return atomicWriteFile(path, body, 0o600) + currentETag := "" + if exists { + currentETag = DocumentETag(current) + } + if !putPreconditionMatches(exists, currentETag, conditions) { + return false, ErrPreconditionFailed + } + if err := atomicWriteFile(path, body, 0o600); err != nil { + return false, err + } + return !exists, nil } -func (s *FileStore) checkAnnotationPagePrecondition(path, ifMatch string) error { +// Delete implements Store. +func (s *FileStore) Delete(_ context.Context, resourceKey, ifMatch string) error { + path, err := s.resourcePath(resourceKey) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + path, err = s.resolveRead(path) + if err != nil { + if errors.Is(err, ErrNotFound) { + return ErrPreconditionFailed + } + return err + } current, err := os.ReadFile(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { - if ifMatch == "*" { - return nil - } return ErrPreconditionFailed } return err } - if ifMatch == "*" { - return ErrPreconditionFailed - } if !IfMatchMatches(ifMatch, DocumentETag(current)) { return ErrPreconditionFailed } - return nil + if err := os.Remove(path); err != nil { + return err + } + return syncDirectory(filepath.Dir(path)) +} + +func (s *FileStore) resourcePath(resourceKey string) (string, error) { + if !validResourceKey(resourceKey) { + return "", ErrNotFound + } + sum := sha256.Sum256([]byte(resourceKey)) + digest := hex.EncodeToString(sum[:]) + return filepath.Join(s.root, "resources", digest[:2], digest+".json"), nil } func atomicWriteFile(path string, body []byte, perm fs.FileMode) error { dir := filepath.Dir(path) - tmp, err := os.CreateTemp(dir, ".annotations-*.tmp") + tmp, err := os.CreateTemp(dir, ".presentation-*.tmp") if err != nil { return err } @@ -147,46 +179,19 @@ func atomicWriteFile(path string, body []byte, perm fs.FileMode) error { return err } cleanup = false - dirHandle, err := os.Open(dir) - if err != nil { - return err - } - defer dirHandle.Close() - return dirHandle.Sync() + return syncDirectory(dir) } -func (s *FileStore) resolve(itemID string, elems ...string) (string, error) { - if itemID == "" || strings.ContainsAny(itemID, "\x00\n\r") { - return "", ErrNotFound - } - cleanItem := filepath.Clean(itemID) - if cleanItem == "." || cleanItem == "/" || cleanItem == ".." || strings.HasPrefix(cleanItem, ".."+string(filepath.Separator)) || filepath.IsAbs(cleanItem) { - return "", ErrNotFound - } - parts := []string{s.root, cleanItem} - for _, elem := range elems { - if elem == "" || strings.ContainsAny(elem, "\x00\n\r") { - return "", ErrNotFound - } - clean := filepath.Clean(elem) - if clean == "." || clean == "/" || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || filepath.IsAbs(clean) { - return "", ErrNotFound - } - parts = append(parts, clean) - } - path := filepath.Join(parts...) - rel, err := filepath.Rel(s.root, path) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return "", ErrNotFound +func syncDirectory(dir string) error { + handle, err := os.Open(dir) + if err != nil { + return err } - return path, nil + defer handle.Close() + return handle.Sync() } -func (s *FileStore) resolveRead(itemID string, elems ...string) (string, error) { - path, err := s.resolve(itemID, elems...) - if err != nil { - return "", err - } +func (s *FileStore) resolveRead(path string) (string, error) { realPath, err := filepath.EvalSymlinks(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { diff --git a/internal/iiif/presentation/v3/store/file_test.go b/internal/iiif/presentation/v3/store/file_test.go index 1b682f3..e806d2e 100644 --- a/internal/iiif/presentation/v3/store/file_test.go +++ b/internal/iiif/presentation/v3/store/file_test.go @@ -5,129 +5,149 @@ import ( "errors" "os" "path/filepath" + "strings" + "sync" "testing" ) -func TestFileStoreGetManifest(t *testing.T) { - root := t.TempDir() - itemDir := filepath.Join(root, "item-1") - if err := os.MkdirAll(itemDir, 0o755); err != nil { +func TestFileStoreResourceLifecycle(t *testing.T) { + st, err := NewFileStore(t.TempDir()) + if err != nil { t.Fatal(err) } - body := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/presentation/v3/item-1/manifest","type":"Manifest","label":{"en":["Item 1"]},"items":[{"id":"http://example.test/presentation/v3/item-1/canvas/1","type":"Canvas"}]}`) - if err := os.WriteFile(filepath.Join(itemDir, "manifest.json"), body, 0o600); err != nil { - t.Fatal(err) + ctx := context.Background() + key := "items/item-1/manifest" + first := []byte(`{"id":"https://example.org/manifest","type":"Manifest"}`) + created, err := st.Put(ctx, key, first, Preconditions{IfNoneMatch: "*"}) + if err != nil || !created { + t.Fatalf("create = %v, %v", created, err) } - - s, err := NewFileStore(root) + document, err := st.Get(ctx, key) if err != nil { t.Fatal(err) } - - got, err := s.GetManifest(context.Background(), "item-1") - if err != nil { - t.Fatal(err) + if string(document.Body) != string(first) { + t.Fatalf("body = %q", document.Body) } - if string(got) != string(body) { - t.Fatalf("body = %q", string(got)) + if document.ModifiedAt.IsZero() { + t.Fatal("missing modification time") } -} -func TestFileStoreGetManifestNotFound(t *testing.T) { - s, err := NewFileStore(t.TempDir()) - if err != nil { - t.Fatal(err) + if _, err := st.Put(ctx, key, first, Preconditions{IfNoneMatch: "*"}); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("duplicate create err = %v", err) } - for _, itemID := range []string{"missing", "../escape", "", "/abs"} { - t.Run(itemID, func(t *testing.T) { - _, err := s.GetManifest(context.Background(), itemID) - if !errors.Is(err, ErrNotFound) { - t.Fatalf("err = %v", err) - } - }) + second := []byte(`{"id":"https://example.org/manifest","type":"Manifest","summary":{"en":["updated"]}}`) + created, err = st.Put(ctx, key, second, Preconditions{IfMatch: DocumentETag(first)}) + if err != nil || created { + t.Fatalf("update = %v, %v", created, err) } -} - -func TestFileStoreAnnotationPageRoundTrip(t *testing.T) { - root := t.TempDir() - itemDir := filepath.Join(root, "item-1") - if err := os.MkdirAll(itemDir, 0o755); err != nil { - t.Fatal(err) + if _, err := st.Put(ctx, key, first, Preconditions{IfMatch: DocumentETag(first)}); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("stale update err = %v", err) } - - s, err := NewFileStore(root) - if err != nil { - t.Fatal(err) + if _, err := st.Put(ctx, key, first, Preconditions{IfMatch: "*"}); err != nil { + t.Fatalf("wildcard update: %v", err) } - - body := []byte(`{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/presentation/v3/item-1/canvas/canvas-1/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/annotations/1","type":"Annotation","textGranularity":"line","motivation":["supplementing"],"body":{"type":"TextualBody","value":"hello"},"target":{"type":"SpecificResource","source":"http://example.test/presentation/v3/item-1/canvas/canvas-1","selector":{"type":"FragmentSelector","value":"xywh=1,2,3,4"}}}]}`) - if err := s.PutAnnotationPage(context.Background(), "item-1", "canvas-1", body, "*"); err != nil { - t.Fatal(err) + if err := st.Delete(ctx, key, DocumentETag(second)); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("stale delete err = %v", err) + } + if err := st.Delete(ctx, key, "*"); err != nil { + t.Fatalf("wildcard delete: %v", err) + } + if _, err := st.Get(ctx, key); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted err = %v", err) } + if err := st.Delete(ctx, key, "*"); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("delete missing err = %v", err) + } +} - got, err := s.GetAnnotationPage(context.Background(), "item-1", "canvas-1") +func TestFileStoreMissingAndInvalidKeys(t *testing.T) { + st, err := NewFileStore(t.TempDir()) if err != nil { t.Fatal(err) } - if string(got) != string(body) { - t.Fatalf("body = %q", string(got)) + ctx := context.Background() + for _, key := range []string{"", strings.Repeat("a", MaxResourceKeyBytes+1), "bad\nkey"} { + if _, err := st.Get(ctx, key); !errors.Is(err, ErrNotFound) { + t.Fatalf("Get(%q) err = %v", key, err) + } + } + if _, err := st.Put(ctx, "missing", []byte(`{}`), Preconditions{IfMatch: "*"}); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("update missing err = %v", err) + } + if _, err := st.Put(ctx, "missing", []byte(`{}`), Preconditions{}); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("unconditional put err = %v", err) } } func TestFileStoreRejectsSymlinkEscape(t *testing.T) { root := t.TempDir() outside := t.TempDir() - if err := os.WriteFile(filepath.Join(outside, "manifest.json"), []byte(`{"secret":true}`), 0o600); err != nil { - t.Fatal(err) - } - if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + if err := os.Symlink(outside, filepath.Join(root, "resources")); err != nil { t.Skipf("symlink unavailable: %v", err) } - s, err := NewFileStore(root) + st, err := NewFileStore(root) if err != nil { t.Fatal(err) } - _, err = s.GetManifest(context.Background(), "link") - if !errors.Is(err, ErrNotFound) { - t.Fatalf("err = %v, want ErrNotFound", err) - } - err = s.PutAnnotationPage(context.Background(), "link", "canvas-1", []byte(`{"type":"AnnotationPage","id":"x","items":[]}`), "*") + _, err = st.Put(context.Background(), "items/one", []byte(`{"secret":true}`), Preconditions{IfNoneMatch: "*"}) if !errors.Is(err, ErrNotFound) { t.Fatalf("put err = %v, want ErrNotFound", err) } - if _, err := os.Stat(filepath.Join(outside, "canvas")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("outside canvas dir err = %v, want not exist", err) - } -} - -func TestFileStoreAnnotationPagePreconditions(t *testing.T) { - root := t.TempDir() - if err := os.MkdirAll(filepath.Join(root, "item-1"), 0o755); err != nil { - t.Fatal(err) - } - s, err := NewFileStore(root) + entries, err := os.ReadDir(outside) if err != nil { t.Fatal(err) } - first := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/annotations","type":"AnnotationPage","items":[]}`) - if err := s.PutAnnotationPage(context.Background(), "item-1", "canvas-1", first, `"missing"`); !errors.Is(err, ErrPreconditionFailed) { - t.Fatalf("missing exact err = %v, want ErrPreconditionFailed", err) - } - if err := s.PutAnnotationPage(context.Background(), "item-1", "canvas-1", first, "*"); err != nil { - t.Fatal(err) - } - if err := s.PutAnnotationPage(context.Background(), "item-1", "canvas-1", first, "*"); !errors.Is(err, ErrPreconditionFailed) { - t.Fatalf("create existing err = %v, want ErrPreconditionFailed", err) - } - second := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/a","type":"Annotation"}]}`) - if err := s.PutAnnotationPage(context.Background(), "item-1", "canvas-1", second, DocumentETag(first)); err != nil { - t.Fatal(err) + if len(entries) != 0 { + t.Fatalf("outside directory was modified: %v", entries) } - got, err := s.GetAnnotationPage(context.Background(), "item-1", "canvas-1") +} + +func TestFileStoreConditionalCreateIsAtomic(t *testing.T) { + st, err := NewFileStore(t.TempDir()) if err != nil { t.Fatal(err) } - if string(got) != string(second) { - t.Fatalf("body = %q", string(got)) + const attempts = 16 + var wg sync.WaitGroup + results := make(chan error, attempts) + for i := 0; i < attempts; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := st.Put(context.Background(), "one/resource", []byte(`{"id":"one"}`), Preconditions{IfNoneMatch: "*"}) + results <- err + }() + } + wg.Wait() + close(results) + succeeded := 0 + preconditionFailed := 0 + for err := range results { + switch { + case err == nil: + succeeded++ + case errors.Is(err, ErrPreconditionFailed): + preconditionFailed++ + default: + t.Fatalf("unexpected create error: %v", err) + } + } + if succeeded != 1 || preconditionFailed != attempts-1 { + t.Fatalf("successes = %d, precondition failures = %d", succeeded, preconditionFailed) + } +} + +func TestIfMatchMatches(t *testing.T) { + etag := `"abc"` + for _, header := range []string{`"abc"`, `"other", "abc"`, "*"} { + if !IfMatchMatches(header, etag) { + t.Fatalf("IfMatchMatches(%q) = false", header) + } + } + for _, header := range []string{"", `W/"abc"`, `"other"`} { + if IfMatchMatches(header, etag) { + t.Fatalf("IfMatchMatches(%q) = true", header) + } } } diff --git a/internal/iiif/presentation/v3/store/mariadb.go b/internal/iiif/presentation/v3/store/mariadb.go index 0d46622..25fb81f 100644 --- a/internal/iiif/presentation/v3/store/mariadb.go +++ b/internal/iiif/presentation/v3/store/mariadb.go @@ -6,8 +6,9 @@ import ( _ "embed" "errors" "fmt" + "time" - _ "github.com/go-sql-driver/mysql" + mysql "github.com/go-sql-driver/mysql" ) type MariaDBStore struct { @@ -49,15 +50,73 @@ func (s *MariaDBStore) Close() error { return s.db.Close() } -func (s *MariaDBStore) GetManifest(ctx context.Context, itemID string) ([]byte, error) { - return getJSON(ctx, s.db, selectManifestSQL, itemID) +// Get implements Store. +func (s *MariaDBStore) Get(ctx context.Context, resourceKey string) (Document, error) { + if !validResourceKey(resourceKey) { + return Document{}, ErrNotFound + } + var body []byte + var modifiedAt time.Time + err := s.db.QueryRowContext(ctx, selectResourceSQL, resourceKey).Scan(&body, &modifiedAt) + if errors.Is(err, sql.ErrNoRows) { + return Document{}, ErrNotFound + } + if err != nil { + return Document{}, err + } + return Document{Body: body, ModifiedAt: modifiedAt.UTC()}, nil } -func (s *MariaDBStore) GetAnnotationPage(ctx context.Context, itemID, canvasID string) ([]byte, error) { - return getJSON(ctx, s.db, selectAnnotationPageSQL, itemID, canvasID) +// Put implements Store. +func (s *MariaDBStore) Put(ctx context.Context, resourceKey string, body []byte, conditions Preconditions) (bool, error) { + if !validResourceKey(resourceKey) { + return false, ErrNotFound + } + if conditions.IfNoneMatch != "" { + if !putPreconditionMatches(false, "", conditions) { + return false, ErrPreconditionFailed + } + if _, err := s.db.ExecContext(ctx, insertResourceSQL, resourceKey, body); err != nil { + if isDuplicateKey(err) { + return false, ErrPreconditionFailed + } + return false, err + } + return true, nil + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer func() { + _ = tx.Rollback() + }() + + var current []byte + err = tx.QueryRowContext(ctx, selectResourceForUpdateSQL, resourceKey).Scan(¤t) + if errors.Is(err, sql.ErrNoRows) { + return false, ErrPreconditionFailed + } + if err != nil { + return false, err + } + if !putPreconditionMatches(true, DocumentETag(current), conditions) { + return false, ErrPreconditionFailed + } + if _, err := tx.ExecContext(ctx, updateResourceSQL, body, resourceKey); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, err + } + return false, nil } -func (s *MariaDBStore) PutAnnotationPage(ctx context.Context, itemID, canvasID string, body []byte, ifMatch string) error { +// Delete implements Store. +func (s *MariaDBStore) Delete(ctx context.Context, resourceKey, ifMatch string) error { + if !validResourceKey(resourceKey) { + return ErrNotFound + } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err @@ -65,64 +124,44 @@ func (s *MariaDBStore) PutAnnotationPage(ctx context.Context, itemID, canvasID s defer func() { _ = tx.Rollback() }() - var current []byte - err = tx.QueryRowContext(ctx, selectAnnotationPageForUpdateSQL, itemID, canvasID).Scan(¤t) + err = tx.QueryRowContext(ctx, selectResourceForUpdateSQL, resourceKey).Scan(¤t) if errors.Is(err, sql.ErrNoRows) { - if ifMatch != "*" { - return ErrPreconditionFailed - } - if _, err := tx.ExecContext(ctx, insertAnnotationPageSQL, itemID, canvasID, body); err != nil { - return err - } - return tx.Commit() + return ErrPreconditionFailed } if err != nil { return err } - if ifMatch == "*" || !IfMatchMatches(ifMatch, DocumentETag(current)) { + if !IfMatchMatches(ifMatch, DocumentETag(current)) { return ErrPreconditionFailed } - if _, err := tx.ExecContext(ctx, updateAnnotationPageSQL, body, itemID, canvasID); err != nil { + if _, err := tx.ExecContext(ctx, deleteResourceSQL, resourceKey); err != nil { return err } return tx.Commit() } -func getJSON(ctx context.Context, db *sql.DB, query string, args ...any) ([]byte, error) { - var body []byte - err := db.QueryRowContext(ctx, query, args...).Scan(&body) - if errors.Is(err, sql.ErrNoRows) { - return nil, ErrNotFound - } - if err != nil { - return nil, err - } - return body, nil +func isDuplicateKey(err error) bool { + var mysqlErr *mysql.MySQLError + return errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 } -//go:embed sql/schema/001_manifests.sql -var createManifestsTableSQL string +//go:embed sql/schema/001_resources.sql +var createResourcesTableSQL string -//go:embed sql/schema/002_annotation_pages.sql -var createAnnotationPagesTableSQL string +//go:embed sql/queries/select_resource.sql +var selectResourceSQL string -//go:embed sql/queries/select_manifest.sql -var selectManifestSQL string +//go:embed sql/queries/select_resource_for_update.sql +var selectResourceForUpdateSQL string -//go:embed sql/queries/select_annotation_page.sql -var selectAnnotationPageSQL string +//go:embed sql/queries/insert_resource.sql +var insertResourceSQL string -//go:embed sql/queries/insert_annotation_page.sql -var insertAnnotationPageSQL string +//go:embed sql/queries/update_resource.sql +var updateResourceSQL string -//go:embed sql/queries/update_annotation_page.sql -var updateAnnotationPageSQL string +//go:embed sql/queries/delete_resource.sql +var deleteResourceSQL string -//go:embed sql/queries/select_annotation_page_for_update.sql -var selectAnnotationPageForUpdateSQL string - -var mariadbSchemaStatements = []string{ - createManifestsTableSQL, - createAnnotationPagesTableSQL, -} +var mariadbSchemaStatements = []string{createResourcesTableSQL} diff --git a/internal/iiif/presentation/v3/store/mariadb_test.go b/internal/iiif/presentation/v3/store/mariadb_test.go index 4dbecfa..d47435a 100644 --- a/internal/iiif/presentation/v3/store/mariadb_test.go +++ b/internal/iiif/presentation/v3/store/mariadb_test.go @@ -5,11 +5,22 @@ import ( "database/sql" "errors" "os" + "strings" + "sync" "testing" _ "github.com/go-sql-driver/mysql" ) +func TestMariaDBSchemaHasNoForeignKeys(t *testing.T) { + normalized := strings.ToLower(createResourcesTableSQL) + for _, forbidden := range []string{"foreign key", "references"} { + if strings.Contains(normalized, forbidden) { + t.Fatalf("Presentation schema contains %q", forbidden) + } + } +} + func TestMariaDBStoreIntegration(t *testing.T) { dsn := os.Getenv("TEST_DSN") if dsn == "" { @@ -30,36 +41,62 @@ func TestMariaDBStoreIntegration(t *testing.T) { t.Fatal(err) } defer db.Close() + _, _ = db.ExecContext(ctx, `DELETE FROM iiif_presentation_resources WHERE resource_key LIKE 'triplet-test/%'`) - _, _ = db.ExecContext(ctx, `DELETE FROM iiif_presentation_annotation_pages WHERE item_id = 'item-1'`) - _, _ = db.ExecContext(ctx, `DELETE FROM iiif_presentation_manifests WHERE item_id = 'item-1'`) - - manifest := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/presentation/v3/item-1/manifest","type":"Manifest","label":{"en":["Item 1"]},"items":[]}`) - if _, err := db.ExecContext(ctx, `INSERT INTO iiif_presentation_manifests (item_id, body) VALUES (?, ?)`, "item-1", manifest); err != nil { - t.Fatal(err) + key := "triplet-test/items/item-1/manifest" + first := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/manifest","type":"Manifest","label":{"en":["Item 1"]},"items":[]}`) + created, err := st.Put(ctx, key, first, Preconditions{IfNoneMatch: "*"}) + if err != nil || !created { + t.Fatalf("create = %v, %v", created, err) } - got, err := st.GetManifest(ctx, "item-1") + document, err := st.Get(ctx, key) if err != nil { t.Fatal(err) } - if string(got) != string(manifest) { - t.Fatalf("manifest = %q", string(got)) + if string(document.Body) != string(first) || document.ModifiedAt.IsZero() { + t.Fatalf("document = %#v", document) } - - page := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/presentation/v3/item-1/canvas/canvas-1/annotations","type":"AnnotationPage","items":[]}`) - if err := st.PutAnnotationPage(ctx, "item-1", "canvas-1", page, "*"); err != nil { - t.Fatal(err) + second := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/manifest","type":"Manifest","label":{"en":["Item 1"]},"items":[],"summary":{"en":["updated"]}}`) + created, err = st.Put(ctx, key, second, Preconditions{IfMatch: DocumentETag(first)}) + if err != nil || created { + t.Fatalf("update = %v, %v", created, err) } - got, err = st.GetAnnotationPage(ctx, "item-1", "canvas-1") - if err != nil { - t.Fatal(err) + if _, err := st.Put(ctx, key, first, Preconditions{IfMatch: DocumentETag(first)}); !errors.Is(err, ErrPreconditionFailed) { + t.Fatalf("stale update err = %v", err) } - if string(got) != string(page) { - t.Fatalf("page = %q", string(got)) + if err := st.Delete(ctx, key, DocumentETag(second)); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := st.Get(ctx, key); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted err = %v", err) } - _, err = st.GetManifest(ctx, "missing") - if !errors.Is(err, ErrNotFound) { - t.Fatalf("err = %v", err) + concurrentKey := "triplet-test/concurrent-create" + const attempts = 8 + var wait sync.WaitGroup + results := make(chan error, attempts) + for i := 0; i < attempts; i++ { + wait.Add(1) + go func() { + defer wait.Done() + _, err := st.Put(ctx, concurrentKey, first, Preconditions{IfNoneMatch: "*"}) + results <- err + }() + } + wait.Wait() + close(results) + succeeded := 0 + failed := 0 + for err := range results { + if err == nil { + succeeded++ + } else if errors.Is(err, ErrPreconditionFailed) { + failed++ + } else { + t.Fatalf("concurrent create: %v", err) + } + } + if succeeded != 1 || failed != attempts-1 { + t.Fatalf("concurrent creates: succeeded=%d failed=%d", succeeded, failed) } } diff --git a/internal/iiif/presentation/v3/store/sql/queries/delete_resource.sql b/internal/iiif/presentation/v3/store/sql/queries/delete_resource.sql new file mode 100644 index 0000000..abfa6e9 --- /dev/null +++ b/internal/iiif/presentation/v3/store/sql/queries/delete_resource.sql @@ -0,0 +1,2 @@ +DELETE FROM iiif_presentation_resources +WHERE resource_key = ? diff --git a/internal/iiif/presentation/v3/store/sql/queries/insert_annotation_page.sql b/internal/iiif/presentation/v3/store/sql/queries/insert_annotation_page.sql deleted file mode 100644 index 32aa89c..0000000 --- a/internal/iiif/presentation/v3/store/sql/queries/insert_annotation_page.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO iiif_presentation_annotation_pages (item_id, canvas_id, body) -VALUES (?, ?, ?) diff --git a/internal/iiif/presentation/v3/store/sql/queries/insert_resource.sql b/internal/iiif/presentation/v3/store/sql/queries/insert_resource.sql new file mode 100644 index 0000000..63d4a0a --- /dev/null +++ b/internal/iiif/presentation/v3/store/sql/queries/insert_resource.sql @@ -0,0 +1,2 @@ +INSERT INTO iiif_presentation_resources (resource_key, body) +VALUES (?, ?) diff --git a/internal/iiif/presentation/v3/store/sql/queries/select_annotation_page.sql b/internal/iiif/presentation/v3/store/sql/queries/select_annotation_page.sql deleted file mode 100644 index 48f6d5c..0000000 --- a/internal/iiif/presentation/v3/store/sql/queries/select_annotation_page.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT body FROM iiif_presentation_annotation_pages WHERE item_id = ? AND canvas_id = ? diff --git a/internal/iiif/presentation/v3/store/sql/queries/select_annotation_page_for_update.sql b/internal/iiif/presentation/v3/store/sql/queries/select_annotation_page_for_update.sql deleted file mode 100644 index 126b0d7..0000000 --- a/internal/iiif/presentation/v3/store/sql/queries/select_annotation_page_for_update.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT body FROM iiif_presentation_annotation_pages -WHERE item_id = ? AND canvas_id = ? -FOR UPDATE diff --git a/internal/iiif/presentation/v3/store/sql/queries/select_manifest.sql b/internal/iiif/presentation/v3/store/sql/queries/select_manifest.sql deleted file mode 100644 index 524aae5..0000000 --- a/internal/iiif/presentation/v3/store/sql/queries/select_manifest.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT body FROM iiif_presentation_manifests WHERE item_id = ? diff --git a/internal/iiif/presentation/v3/store/sql/queries/select_resource.sql b/internal/iiif/presentation/v3/store/sql/queries/select_resource.sql new file mode 100644 index 0000000..e07bfe7 --- /dev/null +++ b/internal/iiif/presentation/v3/store/sql/queries/select_resource.sql @@ -0,0 +1,3 @@ +SELECT body, updated_at +FROM iiif_presentation_resources +WHERE resource_key = ? diff --git a/internal/iiif/presentation/v3/store/sql/queries/select_resource_for_update.sql b/internal/iiif/presentation/v3/store/sql/queries/select_resource_for_update.sql new file mode 100644 index 0000000..0d662e2 --- /dev/null +++ b/internal/iiif/presentation/v3/store/sql/queries/select_resource_for_update.sql @@ -0,0 +1,4 @@ +SELECT body +FROM iiif_presentation_resources +WHERE resource_key = ? +FOR UPDATE diff --git a/internal/iiif/presentation/v3/store/sql/queries/update_annotation_page.sql b/internal/iiif/presentation/v3/store/sql/queries/update_annotation_page.sql deleted file mode 100644 index 4b90ce1..0000000 --- a/internal/iiif/presentation/v3/store/sql/queries/update_annotation_page.sql +++ /dev/null @@ -1,3 +0,0 @@ -UPDATE iiif_presentation_annotation_pages -SET body = ?, updated_at = CURRENT_TIMESTAMP(6) -WHERE item_id = ? AND canvas_id = ? diff --git a/internal/iiif/presentation/v3/store/sql/queries/update_resource.sql b/internal/iiif/presentation/v3/store/sql/queries/update_resource.sql new file mode 100644 index 0000000..7294cc7 --- /dev/null +++ b/internal/iiif/presentation/v3/store/sql/queries/update_resource.sql @@ -0,0 +1,3 @@ +UPDATE iiif_presentation_resources +SET body = ?, updated_at = CURRENT_TIMESTAMP(6) +WHERE resource_key = ? diff --git a/internal/iiif/presentation/v3/store/sql/schema/001_manifests.sql b/internal/iiif/presentation/v3/store/sql/schema/001_manifests.sql deleted file mode 100644 index 2b265a7..0000000 --- a/internal/iiif/presentation/v3/store/sql/schema/001_manifests.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE IF NOT EXISTS iiif_presentation_manifests ( - item_id varchar(255) NOT NULL, - body json NOT NULL, - created_at timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), - updated_at timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), - PRIMARY KEY (item_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci diff --git a/internal/iiif/presentation/v3/store/sql/schema/001_resources.sql b/internal/iiif/presentation/v3/store/sql/schema/001_resources.sql new file mode 100644 index 0000000..f2214f5 --- /dev/null +++ b/internal/iiif/presentation/v3/store/sql/schema/001_resources.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS iiif_presentation_resources ( + resource_key varchar(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + body longblob NOT NULL, + created_at datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + PRIMARY KEY (resource_key) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci diff --git a/internal/iiif/presentation/v3/store/sql/schema/002_annotation_pages.sql b/internal/iiif/presentation/v3/store/sql/schema/002_annotation_pages.sql deleted file mode 100644 index a70412b..0000000 --- a/internal/iiif/presentation/v3/store/sql/schema/002_annotation_pages.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE IF NOT EXISTS iiif_presentation_annotation_pages ( - item_id varchar(255) NOT NULL, - canvas_id varchar(255) NOT NULL, - body json NOT NULL, - created_at timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), - updated_at timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), - PRIMARY KEY (item_id, canvas_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci diff --git a/internal/iiif/presentation/v3/store/store.go b/internal/iiif/presentation/v3/store/store.go index 54f22dc..315fa8d 100644 --- a/internal/iiif/presentation/v3/store/store.go +++ b/internal/iiif/presentation/v3/store/store.go @@ -6,20 +6,38 @@ import ( "encoding/hex" "errors" "strings" + "time" ) -// ErrNotFound is returned when a manifest does not exist. +const MaxResourceKeyBytes = 768 + +// ErrNotFound is returned when a Presentation resource does not exist. var ErrNotFound = errors.New("presentation store: not found") -// ErrPreconditionFailed is returned when a conditional write precondition does -// not match the current stored representation. +// ErrPreconditionFailed is returned when a conditional mutation precondition +// does not match the current stored representation. var ErrPreconditionFailed = errors.New("presentation store: precondition failed") -// Store reads Presentation API documents by item id. +// Document is the byte-exact stored representation of a Presentation resource. +type Document struct { + Body []byte + ModifiedAt time.Time +} + +// Preconditions contains the HTTP validators used for an atomic PUT. +// Handlers require exactly one of IfMatch or IfNoneMatch. +type Preconditions struct { + IfMatch string + IfNoneMatch string +} + +// Store persists arbitrary Presentation API resources by their normalized +// relative public path. Application-specific identifiers and policy stay +// outside this protocol-oriented boundary. type Store interface { - GetManifest(ctx context.Context, itemID string) ([]byte, error) - GetAnnotationPage(ctx context.Context, itemID, canvasID string) ([]byte, error) - PutAnnotationPage(ctx context.Context, itemID, canvasID string, body []byte, ifMatch string) error + Get(ctx context.Context, resourceKey string) (Document, error) + Put(ctx context.Context, resourceKey string, body []byte, conditions Preconditions) (created bool, err error) + Delete(ctx context.Context, resourceKey, ifMatch string) error } // DocumentETag returns the strong ETag used for stored Presentation documents. @@ -28,11 +46,29 @@ func DocumentETag(body []byte) string { return `"` + hex.EncodeToString(sum[:]) + `"` } +// IfMatchMatches reports whether an If-Match field allows the current strong +// entity tag. Weak validators never satisfy If-Match. func IfMatchMatches(header, etag string) bool { + header = strings.TrimSpace(header) + if header == "*" { + return true + } for _, candidate := range strings.Split(header, ",") { - if strings.TrimSpace(candidate) == etag { + candidate = strings.TrimSpace(candidate) + if !strings.HasPrefix(candidate, "W/") && candidate == etag { return true } } return false } + +func validResourceKey(key string) bool { + return key != "" && len(key) <= MaxResourceKeyBytes && !strings.ContainsAny(key, "\x00\n\r") +} + +func putPreconditionMatches(exists bool, currentETag string, conditions Preconditions) bool { + if conditions.IfNoneMatch != "" { + return conditions.IfMatch == "" && conditions.IfNoneMatch == "*" && !exists + } + return conditions.IfMatch != "" && exists && IfMatchMatches(conditions.IfMatch, currentETag) +} diff --git a/internal/iiif/presentation/v3/validate/validate.go b/internal/iiif/presentation/v3/validate/validate.go index fb7894b..9c74fac 100644 --- a/internal/iiif/presentation/v3/validate/validate.go +++ b/internal/iiif/presentation/v3/validate/validate.go @@ -4,252 +4,68 @@ import ( "encoding/json" "errors" "fmt" - "slices" + "net/url" "strings" + textgranularityschema "github.com/libops/iiif-spec/extension/textgranularity/schema" + presentationschema "github.com/libops/iiif-spec/presentation/v3/schema" "github.com/libops/triplet/internal/iiif/presentation/v3/types" ) -// ValidateManifestBytes performs structural validation for the manifest -// surface triplet serves today. It is intentionally narrower than the full -// Presentation 3 validator, but it enforces the required top-level contract. -func ValidateManifestBytes(b []byte) error { - var doc manifestDocument - if err := json.Unmarshal(b, &doc); err != nil { - return fmt.Errorf("decode manifest: %w", err) - } - if !hasContext(doc.Context, types.Context) { - return fmt.Errorf("manifest @context must include %q", types.Context) - } - if strings.TrimSpace(doc.ID) == "" { - return errors.New("manifest id is required") - } - if doc.Type != types.TypeManifest { - return fmt.Errorf("manifest type must be %q", types.TypeManifest) - } - if len(doc.Items) == 0 { - return errors.New("manifest items is required") - } - return nil -} - -// ValidateAnnotationPageBytes validates the annotation-page authoring surface -// triplet supports. It enforces the required top-level fields plus the text -// granularity extension constraints triplet explicitly supports. -func ValidateAnnotationPageBytes(b []byte) error { - var doc annotationPageDocument - if err := json.Unmarshal(b, &doc); err != nil { - return fmt.Errorf("decode annotation page: %w", err) - } - if !hasContext(doc.Context, types.Context) { - return fmt.Errorf("annotation page @context must include %q", types.Context) - } - if strings.TrimSpace(doc.ID) == "" { - return errors.New("annotation page id is required") - } - if doc.Type != types.TypeAnnotationPage { - return fmt.Errorf("annotation page type must be %q", types.TypeAnnotationPage) - } - if doc.Items == nil { - return errors.New("annotation page items is required") - } - - needsTextGranularityContext := false - for i, item := range doc.Items { - if err := item.validate(); err != nil { - return fmt.Errorf("annotation page items[%d]: %w", i, err) - } - if item.TextGranularity != "" { - needsTextGranularityContext = true - } - } - if needsTextGranularityContext && !hasContext(doc.Context, types.ContextTextGranularity) { - return fmt.Errorf("annotation page @context must include %q when textGranularity is used", types.ContextTextGranularity) - } - return nil -} - -type manifestDocument struct { - Context any `json:"@context"` - ID string `json:"id"` - Type string `json:"type"` - Items []json.RawMessage `json:"items"` -} - -type annotationPageDocument struct { - Context any `json:"@context"` - ID string `json:"id"` - Type string `json:"type"` - Items []annotationDocument `json:"items"` -} - -type annotationDocument struct { - ID string `json:"id"` - Type string `json:"type"` - Motivation motivationValue `json:"motivation"` - Body json.RawMessage `json:"body"` - Target json.RawMessage `json:"target"` - TextGranularity string `json:"textGranularity,omitempty"` -} - -func (a annotationDocument) validate() error { - if strings.TrimSpace(a.ID) == "" { - return errors.New("id is required") - } - if a.Type != types.TypeAnnotation { - return fmt.Errorf("type must be %q", types.TypeAnnotation) - } - if len(a.Body) == 0 { - return errors.New("body is required") - } - if len(a.Target) == 0 { - return errors.New("target is required") - } - if err := validateBody(a.Body); err != nil { - return fmt.Errorf("body: %w", err) - } - if err := validateTarget(a.Target); err != nil { - return fmt.Errorf("target: %w", err) - } - if a.TextGranularity != "" { - if !slices.Contains(validTextGranularity, a.TextGranularity) { - return fmt.Errorf("textGranularity %q is not supported", a.TextGranularity) - } - if !a.Motivation.contains(types.MotivationSupplementing) { - return fmt.Errorf("textGranularity requires motivation %q", types.MotivationSupplementing) - } - } - return nil -} - -func validateBody(raw json.RawMessage) error { - if len(raw) == 0 { - return errors.New("missing") - } - - var single map[string]any - if err := json.Unmarshal(raw, &single); err == nil && len(single) > 0 { - return validateBodyObject(single) - } - - var many []map[string]any - if err := json.Unmarshal(raw, &many); err != nil { - return errors.New("must be an object or array") - } - if len(many) == 0 { - return errors.New("must not be empty") - } - for i, body := range many { - if err := validateBodyObject(body); err != nil { - return fmt.Errorf("[%d]: %w", i, err) - } - } - return nil -} - -func validateBodyObject(body map[string]any) error { - typ, _ := body["type"].(string) - switch typ { - case types.TypeTextualBody: - if strings.TrimSpace(stringValue(body["value"])) == "" { - return errors.New("TextualBody value is required") - } - return nil - case types.TypeSpecificResource: - if strings.TrimSpace(stringValue(body["source"])) == "" { - return errors.New("SpecificResource source is required") - } - return nil - default: - if typ == "" { - return errors.New("type is required") - } - if id := stringValue(body["id"]); strings.TrimSpace(id) == "" { - return errors.New("body resource id is required for non-TextualBody bodies") - } - return nil - } +// Resource identifies a validated top-level IIIF Presentation resource. +type Resource struct { + ID string + Type string } -func validateTarget(raw json.RawMessage) error { - var target map[string]any - if err := json.Unmarshal(raw, &target); err != nil { - return errors.New("must be an object") - } - typ := stringValue(target["type"]) - switch typ { +// ValidateResourceBytes validates a byte-exact top-level Presentation API +// resource with libops/iiif-spec's extension-aware schemas. Text Granularity +// annotations use the extension's generic schema; application-specific OCR +// profiles do not belong at this protocol boundary. +func ValidateResourceBytes(body []byte) (Resource, error) { + var resource Resource + if err := json.Unmarshal(body, &resource); err != nil { + return Resource{}, fmt.Errorf("decode presentation resource: %w", err) + } + if strings.TrimSpace(resource.ID) == "" { + return Resource{}, errors.New("presentation resource id is required") + } + if err := validateHTTPID(resource.ID); err != nil { + return Resource{}, err + } + + var err error + switch resource.Type { + case types.TypeManifest: + err = presentationschema.ValidateExtensibleManifestBytes(body) + case types.TypeCanvas: + err = presentationschema.ValidateExtensibleCanvasBytes(body) + case types.TypeAnnotationPage: + err = textgranularityschema.ValidateAnnotationPageBytes(body) + case types.TypeAnnotation: + err = textgranularityschema.ValidateAnnotationBytes(body) + case types.TypeRange: + err = presentationschema.ValidateExtensibleRangeBytes(body) + case types.TypeCollection, types.TypeAnnotationCollection: + err = presentationschema.ValidateExtensibleBytes(body) case "": - if strings.TrimSpace(stringValue(target["id"])) == "" { - return errors.New("id is required") - } - case types.TypeSpecificResource: - if strings.TrimSpace(stringValue(target["source"])) == "" { - return errors.New("SpecificResource source is required") - } + return Resource{}, errors.New("presentation resource type is required") default: - if strings.TrimSpace(stringValue(target["id"])) == "" { - return errors.New("id is required") - } - } - return nil -} - -type motivationValue []string - -func (m *motivationValue) UnmarshalJSON(data []byte) error { - if string(data) == "null" { - *m = nil - return nil + return Resource{}, fmt.Errorf("presentation resource type %q is not supported", resource.Type) } - var single string - if err := json.Unmarshal(data, &single); err == nil { - *m = []string{single} - return nil + if err != nil { + return Resource{}, err } - var many []string - if err := json.Unmarshal(data, &many); err != nil { - return err - } - *m = many - return nil + return resource, nil } -func (m motivationValue) contains(needle string) bool { - for _, v := range m { - if v == needle { - return true - } +func validateHTTPID(id string) error { + parsed, err := url.Parse(id) + if err != nil { + return fmt.Errorf("presentation resource id: %w", err) } - return false -} - -func hasContext(ctx any, want string) bool { - switch v := ctx.(type) { - case string: - return v == want - case []any: - for _, item := range v { - if s, ok := item.(string); ok && s == want { - return true - } - } - case map[string]any: - _, ok := v[want] - return ok + if (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return fmt.Errorf("presentation resource id must be an absolute http(s) URL") } - return false -} - -func stringValue(v any) string { - s, _ := v.(string) - return s -} - -var validTextGranularity = []string{ - types.TextGranularityPage, - types.TextGranularityBlock, - types.TextGranularityParagraph, - types.TextGranularityLine, - types.TextGranularityWord, - types.TextGranularityGlyph, + return nil } diff --git a/internal/iiif/presentation/v3/validate/validate_test.go b/internal/iiif/presentation/v3/validate/validate_test.go index 100e14c..eff7104 100644 --- a/internal/iiif/presentation/v3/validate/validate_test.go +++ b/internal/iiif/presentation/v3/validate/validate_test.go @@ -2,26 +2,80 @@ package validate import "testing" -func TestValidateManifestBytes(t *testing.T) { - valid := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/manifest","type":"Manifest","label":{"en":["Manifest"]},"items":[{"id":"http://example.test/canvas/1","type":"Canvas"}]}`) - if err := ValidateManifestBytes(valid); err != nil { - t.Fatalf("unexpected error: %v", err) +func TestValidateResourceBytes(t *testing.T) { + tests := []struct { + name string + doc string + typeName string + }{ + { + name: "manifest", + doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/manifest","type":"Manifest","label":{"en":["Manifest"]},"items":[]}`, + typeName: "Manifest", + }, + { + name: "canvas", + doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/canvas/1","type":"Canvas","width":100,"height":200,"items":[]}`, + typeName: "Canvas", + }, + { + name: "annotation page with compact target and extension property", + doc: `{"@context":["https://example.org/extension/context.json","http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"https://example.org/pages/1","type":"AnnotationPage","items":[{"id":"https://example.org/annotations/1","type":"Annotation","motivation":"supplementing","body":{"type":"TextualBody","value":"hello"},"target":"https://example.org/canvas/1#xywh=1,2,3,4","textGranularity":"line"}],"example:custom":7}`, + typeName: "AnnotationPage", + }, + { + name: "standalone annotation", + doc: `{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"https://example.org/annotations/1","type":"Annotation","target":"https://example.org/canvas/1","textGranularity":"token"}`, + typeName: "Annotation", + }, + { + name: "collection", + doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/collection","type":"Collection","label":{"en":["Collection"]},"items":[]}`, + typeName: "Collection", + }, + { + name: "annotation collection", + doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/annotation-collection","type":"AnnotationCollection"}`, + typeName: "AnnotationCollection", + }, + { + name: "range", + doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/range/1","type":"Range","label":{"en":["Chapter one"]},"items":[{"id":"https://example.org/canvas/1","type":"Canvas"}]}`, + typeName: "Range", + }, } - - invalid := []byte(`{"id":"http://example.test/manifest","type":"Manifest","items":[]}`) - if err := ValidateManifestBytes(invalid); err == nil { - t.Fatal("expected validation error") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resource, err := ValidateResourceBytes([]byte(test.doc)) + if err != nil { + t.Fatalf("valid resource rejected: %v", err) + } + if resource.Type != test.typeName { + t.Fatalf("type = %q, want %q", resource.Type, test.typeName) + } + }) } } -func TestValidateAnnotationPageBytes(t *testing.T) { - valid := []byte(`{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"http://example.test/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/annotations/1","type":"Annotation","textGranularity":"line","motivation":["supplementing"],"body":{"type":"TextualBody","value":"hello"},"target":{"type":"SpecificResource","source":"http://example.test/canvas/1","selector":{"type":"FragmentSelector","value":"xywh=1,2,3,4"}}}]}`) - if err := ValidateAnnotationPageBytes(valid); err != nil { - t.Fatalf("unexpected error: %v", err) +func TestValidateResourceBytesRejectsInvalidDocuments(t *testing.T) { + tests := []struct { + name string + doc string + }{ + {name: "malformed JSON", doc: `{`}, + {name: "missing id", doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","type":"AnnotationPage","items":[]}`}, + {name: "non HTTP id", doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"urn:example:1","type":"AnnotationPage","items":[]}`}, + {name: "missing context", doc: `{"id":"https://example.org/pages/1","type":"AnnotationPage","items":[]}`}, + {name: "Presentation context not final", doc: `{"@context":["http://iiif.io/api/presentation/3/context.json","https://example.org/extension/context.json"],"id":"https://example.org/pages/1","type":"AnnotationPage","items":[]}`}, + {name: "text granularity context missing", doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/pages/1","type":"AnnotationPage","items":[{"id":"https://example.org/annotations/1","type":"Annotation","textGranularity":"line"}]}`}, + {name: "text granularity is not string", doc: `{"@context":["http://iiif.io/api/extension/text-granularity/context.json","http://iiif.io/api/presentation/3/context.json"],"id":"https://example.org/pages/1","type":"AnnotationPage","items":[{"id":"https://example.org/annotations/1","type":"Annotation","textGranularity":["line"]}]}`}, + {name: "unsupported top-level type", doc: `{"@context":"http://iiif.io/api/presentation/3/context.json","id":"https://example.org/sequence/1","type":"Sequence","items":[]}`}, } - - invalid := []byte(`{"@context":"http://iiif.io/api/presentation/3/context.json","id":"http://example.test/annotations","type":"AnnotationPage","items":[{"id":"http://example.test/annotations/1","type":"Annotation","textGranularity":"line","body":{"type":"TextualBody","value":"hello"},"target":{"id":"http://example.test/canvas/1"}}]}`) - if err := ValidateAnnotationPageBytes(invalid); err == nil { - t.Fatal("expected validation error") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := ValidateResourceBytes([]byte(test.doc)); err == nil { + t.Fatal("invalid resource accepted") + } + }) } } diff --git a/internal/server/server.go b/internal/server/server.go index e7e2d95..fdd155c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -124,8 +124,9 @@ func Build(cfg *config.Config, logger *slog.Logger) (*http.Server, error) { } h := preshandler.New( cfg.IIIF.Presentation.Prefix, + cfg.Server.PublicBaseURL, st, - cors.New(cfg.IIIF.AllowedOrigins, "ETag"), + cors.New(cfg.IIIF.AllowedOrigins, "ETag, Last-Modified, Content-Length, Location"), cfg.IIIF.Presentation.WriteEnabled, cfg.IIIF.Presentation.WriteToken, logger, @@ -264,6 +265,7 @@ func buildSource(cfg *config.Config, logger *slog.Logger) (storage.Opener, func( int64(cfg.Sources.HTTP.MaxBytes), ) op.AllowPrivateHosts = cfg.Sources.HTTP.AllowPrivateHosts + op.ForwardAuthHeaders = cfg.Sources.HTTP.ForwardAuthHeaders authOp := storage.NewHTTPOpener( cfg.Sources.HTTP.AllowedOrigins, cfg.Sources.HTTP.RequestTimeout, diff --git a/internal/storage/http.go b/internal/storage/http.go index 4d76090..a5abeb8 100644 --- a/internal/storage/http.go +++ b/internal/storage/http.go @@ -7,12 +7,15 @@ import ( "io" "net" "net/http" + "net/netip" "net/url" "os" "strconv" "strings" "sync" "time" + + "github.com/libops/triplet/internal/redact" ) // HTTPOpener resolves identifiers as URL-encoded HTTP(S) URLs. @@ -83,7 +86,7 @@ func (h *HTTPOpener) Open(ctx context.Context, identifier string) (io.ReadSeekCl resp, err := h.client().Do(req) if err != nil { - return nil, Meta{}, fmt.Errorf("http source fetch %q: %w", target.Redacted(), err) + return nil, Meta{}, fmt.Errorf("http source fetch %q: %w", safeTarget(target), err) } defer resp.Body.Close() @@ -94,7 +97,7 @@ func (h *HTTPOpener) Open(ctx context.Context, identifier string) (io.ReadSeekCl case http.StatusUnauthorized, http.StatusForbidden: return nil, Meta{}, fmt.Errorf("%w: upstream status %d", ErrForbidden, resp.StatusCode) default: - return nil, Meta{}, fmt.Errorf("http source %q: upstream status %d", target.Redacted(), resp.StatusCode) + return nil, Meta{}, fmt.Errorf("http source %q: upstream status %d", safeTarget(target), resp.StatusCode) } var reader io.Reader = resp.Body @@ -117,7 +120,7 @@ func (h *HTTPOpener) Open(ctx context.Context, identifier string) (io.ReadSeekCl } if h.MaxBytes > 0 && n > h.MaxBytes { cleanup() - return nil, Meta{}, fmt.Errorf("http source %q: response exceeds max_bytes %d", target.Redacted(), h.MaxBytes) + return nil, Meta{}, fmt.Errorf("http source %q: response exceeds max_bytes %d", safeTarget(target), h.MaxBytes) } if _, err := tmp.Seek(0, io.SeekStart); err != nil { cleanup() @@ -144,7 +147,7 @@ func (h *HTTPOpener) Meta(ctx context.Context, identifier string) (Meta, error) resp, err := h.client().Do(req) if err != nil { - return Meta{}, fmt.Errorf("http source head %q: %w", target.Redacted(), err) + return Meta{}, fmt.Errorf("http source head %q: %w", safeTarget(target), err) } defer resp.Body.Close() switch resp.StatusCode { @@ -154,7 +157,7 @@ func (h *HTTPOpener) Meta(ctx context.Context, identifier string) (Meta, error) size = resp.ContentLength } if h.MaxBytes > 0 && size > h.MaxBytes { - return Meta{}, fmt.Errorf("http source %q: response exceeds max_bytes %d", target.Redacted(), h.MaxBytes) + return Meta{}, fmt.Errorf("http source %q: response exceeds max_bytes %d", safeTarget(target), h.MaxBytes) } return httpMeta(resp.Header, size), nil case http.StatusMethodNotAllowed, http.StatusNotImplemented: @@ -166,13 +169,13 @@ func (h *HTTPOpener) Meta(ctx context.Context, identifier string) (Meta, error) _ = rc.Close() return meta, nil } - return Meta{}, fmt.Errorf("http source %q: metadata unavailable", target.Redacted()) + return Meta{}, fmt.Errorf("http source %q: metadata unavailable", safeTarget(target)) case http.StatusNotFound: return Meta{}, fmt.Errorf("%w: upstream 404", ErrNotFound) case http.StatusUnauthorized, http.StatusForbidden: return Meta{}, fmt.Errorf("%w: upstream status %d", ErrForbidden, resp.StatusCode) default: - return Meta{}, fmt.Errorf("http source %q: upstream status %d", target.Redacted(), resp.StatusCode) + return Meta{}, fmt.Errorf("http source %q: upstream status %d", safeTarget(target), resp.StatusCode) } } @@ -184,6 +187,9 @@ func (h *HTTPOpener) parseTarget(identifier string) (*url.URL, error) { if err != nil || (target.Scheme != "http" && target.Scheme != "https") { return nil, fmt.Errorf("%w: identifier must be an http(s) URL", ErrNotFound) } + if target.User != nil { + return nil, fmt.Errorf("%w: identifier must not contain URL credentials", ErrNotFound) + } if !h.originAllowed(target) { return nil, fmt.Errorf("%w: origin %q not in allow-list", ErrNotFound, urlOrigin(target)) } @@ -202,7 +208,7 @@ func (h *HTTPOpener) openRange(ctx context.Context, target *url.URL) (io.ReadSee resp, err := h.client().Do(req) if err != nil { - return nil, Meta{}, false, fmt.Errorf("http source range fetch %q: %w", target.Redacted(), err) + return nil, Meta{}, false, fmt.Errorf("http source range fetch %q: %w", safeTarget(target), err) } defer resp.Body.Close() @@ -215,14 +221,14 @@ func (h *HTTPOpener) openRange(ctx context.Context, target *url.URL) (io.ReadSee case http.StatusUnauthorized, http.StatusForbidden: return nil, Meta{}, true, fmt.Errorf("%w: upstream status %d", ErrForbidden, resp.StatusCode) default: - return nil, Meta{}, true, fmt.Errorf("http source %q: upstream status %d", target.Redacted(), resp.StatusCode) + return nil, Meta{}, true, fmt.Errorf("http source %q: upstream status %d", safeTarget(target), resp.StatusCode) } size, ok := parseContentRangeSize(resp.Header.Get("Content-Range")) if !ok { return nil, Meta{}, false, nil } if h.MaxBytes > 0 && size > h.MaxBytes { - return nil, Meta{}, true, fmt.Errorf("http source %q: response exceeds max_bytes %d", target.Redacted(), h.MaxBytes) + return nil, Meta{}, true, fmt.Errorf("http source %q: response exceeds max_bytes %d", safeTarget(target), h.MaxBytes) } meta := Meta{ ContentType: resp.Header.Get("Content-Type"), @@ -341,9 +347,23 @@ func (h *HTTPOpener) checkRedirect(req *http.Request, via []*http.Request) error if !h.originAllowed(req.URL) { return fmt.Errorf("%w: redirect origin %q not in allow-list", ErrNotFound, urlOrigin(req.URL)) } + if len(via) > 0 && urlOrigin(req.URL) != urlOrigin(via[0].URL) { + // Browser credentials are scoped to the original source origin. A + // redirect target must be allowlisted independently, but it must not + // receive the original origin's credentials. + req.Header.Del("Authorization") + req.Header.Del("Cookie") + } return nil } +func safeTarget(target *url.URL) string { + if target == nil { + return "" + } + return redact.Identifier(target.String()) +} + func httpVersion(etag, lastModified string, size int64) string { etag = strings.TrimSpace(etag) if etag != "" { @@ -422,15 +442,36 @@ func urlOrigin(u *url.URL) string { return strings.ToLower(u.Scheme) + "://" + strings.ToLower(u.Host) } +var nonPublicSourcePrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("192.0.0.0/24"), + netip.MustParsePrefix("192.0.2.0/24"), + netip.MustParsePrefix("192.88.99.0/24"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + netip.MustParsePrefix("240.0.0.0/4"), + netip.MustParsePrefix("100::/64"), + netip.MustParsePrefix("2001:2::/48"), + netip.MustParsePrefix("2001:db8::/32"), +} + func privateAddressBlocked(ip net.IP) bool { - return ip == nil || - ip.IsUnspecified() || - ip.IsLoopback() || - ip.IsPrivate() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsInterfaceLocalMulticast() || - ip.IsMulticast() + address, ok := netip.AddrFromSlice(ip) + if !ok { + return true + } + address = address.Unmap() + if !address.IsGlobalUnicast() || address.IsPrivate() || address.IsLoopback() || address.IsLinkLocalUnicast() || address.IsMulticast() || address.IsUnspecified() { + return true + } + for _, prefix := range nonPublicSourcePrefixes { + if prefix.Contains(address) { + return true + } + } + return false } // seekableBytes is a tiny io.ReadSeeker over a []byte without pulling in diff --git a/internal/storage/http_test.go b/internal/storage/http_test.go index 2e3b5a9..a2f1e6b 100644 --- a/internal/storage/http_test.go +++ b/internal/storage/http_test.go @@ -161,6 +161,114 @@ func TestHTTPOpenerRejectsRedirectToDeniedHost(t *testing.T) { } } +func TestHTTPOpenerForwardsOnlyAuthHeadersWhenEnabled(t *testing.T) { + for _, enabled := range []bool{false, true} { + t.Run(strconv.FormatBool(enabled), func(t *testing.T) { + var requests []http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Header.Clone()) + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write([]byte("image")) + })) + defer srv.Close() + + op := NewHTTPOpener([]string{srv.URL}, 5*time.Second, 0) + op.AllowPrivateHosts = true + op.ForwardAuthHeaders = enabled + ctx := ContextWithAuthHeaders(context.Background(), http.Header{ + "Authorization": []string{"Bearer caller-token"}, + "Cookie": []string{"session=caller-session"}, + "X-Arbitrary": []string{"must-not-forward"}, + }) + rc, _, err := op.Open(ctx, srv.URL+"/source") + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(rc) + _ = rc.Close() + if len(requests) == 0 { + t.Fatal("upstream received no requests") + } + for _, headers := range requests { + wantAuthorization := "" + wantCookie := "" + if enabled { + wantAuthorization = "Bearer caller-token" + wantCookie = "session=caller-session" + } + if got := headers.Get("Authorization"); got != wantAuthorization { + t.Fatalf("Authorization = %q, want %q", got, wantAuthorization) + } + if got := headers.Get("Cookie"); got != wantCookie { + t.Fatalf("Cookie = %q, want %q", got, wantCookie) + } + if got := headers.Get("X-Arbitrary"); got != "" { + t.Fatalf("X-Arbitrary was forwarded: %q", got) + } + } + }) + } +} + +func TestHTTPOpenerStripsAuthHeadersOnCrossOriginRedirect(t *testing.T) { + var finalHeaders []http.Header + final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + finalHeaders = append(finalHeaders, r.Header.Clone()) + _, _ = w.Write([]byte("image")) + })) + defer final.Close() + redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, final.URL+"/source", http.StatusTemporaryRedirect) + })) + defer redirector.Close() + + op := NewHTTPOpener([]string{redirector.URL, final.URL}, 5*time.Second, 0) + op.AllowPrivateHosts = true + op.ForwardAuthHeaders = true + ctx := ContextWithAuthHeaders(context.Background(), http.Header{ + "Authorization": []string{"Bearer caller-token"}, + "Cookie": []string{"session=caller-session"}, + }) + rc, _, err := op.Open(ctx, redirector.URL+"/redirect") + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(rc) + _ = rc.Close() + if len(finalHeaders) == 0 { + t.Fatal("redirect target received no requests") + } + for _, headers := range finalHeaders { + if headers.Get("Authorization") != "" || headers.Get("Cookie") != "" { + t.Fatalf("redirect target received credentials: %#v", headers) + } + } +} + +func TestHTTPOpenerRedactsSourceQueryFromErrors(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "failed", http.StatusInternalServerError) + })) + defer srv.Close() + op := NewHTTPOpener([]string{srv.URL}, 5*time.Second, 0) + op.AllowPrivateHosts = true + _, _, err := op.Open(context.Background(), srv.URL+"/source?token=super-secret") + if err == nil { + t.Fatal("upstream failure accepted") + } + if strings.Contains(err.Error(), "super-secret") || !strings.Contains(err.Error(), "redacted") { + t.Fatalf("error did not redact query: %v", err) + } +} + +func TestHTTPOpenerRejectsURLCredentials(t *testing.T) { + op := NewHTTPOpener([]string{"https://repository.example.edu"}, 5*time.Second, 0) + _, _, err := op.Open(context.Background(), "https://user:secret@repository.example.edu/source.tif") + if !errors.Is(err, ErrNotFound) || !strings.Contains(err.Error(), "must not contain URL credentials") { + t.Fatalf("err = %v", err) + } +} + func TestHTTPOpenerUsesRangeRequests(t *testing.T) { body := []byte("0123456789") var ranges []string @@ -289,7 +397,25 @@ func TestHTTPOpenerMetaUsesHEAD(t *testing.T) { } func TestHTTPOpenerBlocksPrivateAddressesByDefault(t *testing.T) { - for _, raw := range []string{"127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.1.1", "169.254.169.254", "::1", "fc00::1", "fe80::1"} { + for _, raw := range []string{ + "0.1.2.3", + "10.0.0.1", + "100.100.100.200", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "192.0.2.1", + "192.168.1.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.1", + "240.0.0.1", + "::1", + "100::1", + "2001:db8::1", + "fc00::1", + "fe80::1", + } { t.Run(raw, func(t *testing.T) { if !privateAddressBlocked(net.ParseIP(raw)) { t.Fatalf("%s was not blocked", raw) diff --git a/scripts/benchmark-iiif.sh b/scripts/benchmark-iiif.sh index a3f41d5..19b1c74 100755 --- a/scripts/benchmark-iiif.sh +++ b/scripts/benchmark-iiif.sh @@ -33,6 +33,8 @@ APPEND_RUN_REPORTS="${BENCH_APPEND_RUN_REPORTS:-1}" NETWORK="triplet-bench-$RUN_ID" TRIPLET_CONTAINER="triplet-bench-triplet-$RUN_ID" +BENCHMARK_TMP_DIR="" +BENCHMARK_HELPER="" mkdir -p "$OUT_DIR/request-lines" "$OUT_DIR/logs" @@ -49,6 +51,10 @@ cleanup() { docker rm -f "$TRIPLET_CONTAINER" >/dev/null 2>&1 || true docker network rm "$NETWORK" >/dev/null 2>&1 || true fi + if [ -n "$BENCHMARK_TMP_DIR" ] && [ -d "$BENCHMARK_TMP_DIR" ]; then + rm -f -- "$BENCHMARK_HELPER" + rmdir -- "$BENCHMARK_TMP_DIR" + fi } trap cleanup EXIT @@ -59,12 +65,14 @@ require() { fi } -urlencode() { - python3 - "$1" <<'PY' -import sys -from urllib.parse import quote -print(quote(sys.argv[1], safe="")) -PY +prepare_benchmark_helper() { + BENCHMARK_TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/triplet-benchmark.XXXXXX")" + BENCHMARK_HELPER="$BENCHMARK_TMP_DIR/triplet-benchmark-helper" + ( + cd "$ROOT_DIR" + go build -o "$BENCHMARK_HELPER" ./cmd/triplet-benchmark-helper + ) + export BENCHMARK_HELPER } wait_http() { @@ -187,32 +195,19 @@ pass="$4" request_name="$5" request_path="$6" -urlencode() { - python3 - "$1" <<'PY' -import sys -from urllib.parse import quote -print(quote(sys.argv[1], safe="")) -PY -} - csv_escape() { local v="${1//\"/\"\"}" printf '"%s"' "$v" } -encoded="$(urlencode "$image")" +encoded="$("$BENCHMARK_HELPER" urlencode "$image")" if [ "$request_path" = "info.json" ]; then url="${base_url%/}/iiif/3/${encoded}/info.json" else url="${base_url%/}/iiif/3/${encoded}/${request_path}" fi -hash="$(python3 - "$image" "$request_name" "$pass" "$server" <<'PY' -import hashlib -import sys -print(hashlib.sha256("\0".join(sys.argv[1:]).encode("utf-8")).hexdigest()) -PY -)" +hash="$("$BENCHMARK_HELPER" hash "$image" "$request_name" "$pass" "$server")" line_file="$OUT_DIR/request-lines/${hash}.csv" set +e @@ -314,216 +309,21 @@ normalize_mode() { write_matrix_summary() { local index="$1" - python3 - "$OUT_ROOT" "$RUN_ID" "$index" <<'PY' -import csv -import json -import statistics -import sys -from pathlib import Path - -out_root = Path(sys.argv[1]) -run_id = sys.argv[2] -index = Path(sys.argv[3]) - -MODE_ORDER = {"uncached": 0, "cached": 1} - -def fmt_ms(value): - return "-" if value is None else f"{value * 1000:.1f}" - -def fmt_cpu_ms(value): - return "-" if value is None else f"{value * 1000:.2f}" - -def fmt_s(value): - return "-" if value is None else f"{value:.2f}" - -def fmt_rate_per_s(count, duration): - if not count or not duration: - return "-" - return f"{count / duration:.1f}" - -def fmt_size(value): - if value is None: - return "-" - units = ["B", "KiB", "MiB", "GiB"] - size = float(value) - unit = units[0] - for unit in units: - if abs(size) < 1024 or unit == units[-1]: - break - size /= 1024 - return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" - -def fmt_rate(ok, total): - if total == 0: - return "-" - return f"{ok}/{total} ({ok / total * 100:.0f}%)" - -def percentile(values, pct): - if not values: - return None - ordered = sorted(values) - index = (len(ordered) - 1) * pct - lower = int(index) - upper = min(lower + 1, len(ordered) - 1) - if lower == upper: - return ordered[lower] - weight = index - lower - return ordered[lower] * (1 - weight) + ordered[upper] * weight - -def read_resources(path): - if not path.exists(): - return {} - with path.open(newline="", encoding="utf-8") as fh: - return {row["server"]: row for row in csv.DictReader(fh)} - -def sort_key(row): - return (MODE_ORDER.get(row["mode"], 99), int(row["concurrency"])) - -summary_rows = [] -overall_rows = [] -triplet_images = set() -for run_json in sorted(out_root.glob(f"{run_id}-*/run.json")): - run_dir = run_json.parent - requests_csv = run_dir / "requests.csv" - if not requests_csv.exists(): - continue - run = json.loads(run_json.read_text(encoding="utf-8")) - triplet_images.add(run.get("triplet_image", "-")) - by_server = {} - with requests_csv.open(newline="", encoding="utf-8") as fh: - for row in csv.DictReader(fh): - stats = by_server.setdefault(row["server"], {"total": 0, "ok": 0, "times": [], "sizes": []}) - stats["total"] += 1 - if row["curl_exit"] == "0" and row["http_code"].startswith("2"): - stats["ok"] += 1 - stats["times"].append(float(row["time_total"])) - stats["sizes"].append(int(float(row["size_download"]))) - resources = read_resources(run_dir / "resource-summary.csv") - triplet = by_server.get("triplet", {"total": 0, "ok": 0, "times": [], "sizes": []}) - triplet_times = triplet["times"] - duration = float(run.get("measured_duration_seconds") or 0) - cpu_per_request = None - try: - if triplet["ok"]: - mean_cpu = float(resources.get("triplet", {}).get("mean_cpu_percent", "")) - cpu_per_request = mean_cpu / 100 * duration / triplet["ok"] - except ValueError: - pass - max_mem = None - try: - max_mem = float(resources.get("triplet", {}).get("max_mem_mib", "")) - except ValueError: - pass - summary_rows.append({ - "mode": run.get("mode", "-"), - "concurrency": str(run.get("concurrency", "-")), - "row": [ - run.get("mode", "-"), - str(run.get("concurrency", "-")), - fmt_rate(triplet["ok"], triplet["total"]), - fmt_s(duration if duration > 0 else None), - fmt_rate_per_s(triplet["ok"], duration), - fmt_ms(percentile(triplet_times, 0.95)), - fmt_ms(percentile(triplet_times, 0.99)), - fmt_cpu_ms(cpu_per_request), - f"{max_mem:.1f}" if max_mem is not None else "-", - ], - }) - for server, stats in sorted(by_server.items()): - times = stats["times"] - sizes = stats["sizes"] - overall_rows.append({ - "mode": run.get("mode", "-"), - "concurrency": str(run.get("concurrency", "-")), - "server": server, - "row": [ - run.get("mode", "-"), - str(run.get("concurrency", "-")), - server, - fmt_rate(stats["ok"], stats["total"]), - fmt_ms(statistics.median(times) if times else None), - fmt_ms(statistics.fmean(times) if times else None), - fmt_size(statistics.fmean(sizes) if sizes else None), - f"[report](../{run_dir.name}/report.md)", - ], - }) - -if summary_rows: - original = index.read_text(encoding="utf-8") - title, _, remainder = original.partition("\n\n") - lines = [ - title, - "", - "## Summary", - "", - f"Triplet image: `{', '.join(sorted(triplet_images))}`", - "", - "| Mode | Concurrency | Triplet OK | Duration s | Req/s | p95 ms | p99 ms | CPU ms/req | Max MiB |", - "| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: |", - ] - for item in sorted(summary_rows, key=sort_key): - lines.append("| " + " | ".join(item["row"]) + " |") - lines.extend([ - "", - "Status reflects Triplet request success. Performance metrics are informational.", - "", - ]) - if remainder: - lines.append(remainder.rstrip()) - lines.append("") - lines.extend([ - "## Overall Summary", - "", - "| Mode | Concurrency | Server | Success | Median ms | Mean ms | Mean bytes | Report |", - "| --- | ---: | --- | --- | ---: | ---: | ---: | --- |", - ]) - for item in sorted(overall_rows, key=lambda row: (sort_key(row), row["server"])): - lines.append("| " + " | ".join(item["row"]) + " |") - index.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") -PY + "$BENCHMARK_HELPER" matrix-summary "$OUT_ROOT" "$RUN_ID" "$index" } append_matrix_reports() { local index="$1" - python3 - "$OUT_ROOT" "$RUN_ID" "$index" <<'PY' -import json -import sys -from pathlib import Path - -out_root = Path(sys.argv[1]) -run_id = sys.argv[2] -index = Path(sys.argv[3]) - -def demote_headings(markdown): - lines = [] - for line in markdown.splitlines(): - if line.startswith("#"): - lines.append("#" + line) - else: - lines.append(line) - return "\n".join(lines).rstrip() + "\n" - -with index.open("a", encoding="utf-8") as out: - out.write("\n## Run Reports\n\n") - for run_json in sorted(out_root.glob(f"{run_id}-*/run.json")): - run_dir = run_json.parent - report = run_dir / "report.md" - if not report.exists(): - continue - run = json.loads(run_json.read_text(encoding="utf-8")) - out.write(f"### {run_dir.name}\n\n") - out.write(f"- Mode: `{run.get('mode', '-')}`\n") - out.write(f"- Concurrency: `{run.get('concurrency', '-')}`\n") - out.write(f"- Directory: `{run_dir}`\n\n") - out.write(demote_headings(report.read_text(encoding="utf-8"))) - out.write("\n") -PY + "$BENCHMARK_HELPER" append-matrix-reports "$OUT_ROOT" "$RUN_ID" "$index" } main() { require docker require curl + require go + require mktemp require python3 + prepare_benchmark_helper if [ "$MATRIX" = "1" ]; then run_matrix @@ -586,7 +386,7 @@ EOF wait_http triplet "http://127.0.0.1:$TRIPLET_PORT/healthz" "$TRIPLET_CONTAINER" - export OUT_DIR CURL_TIMEOUT + export OUT_DIR CURL_TIMEOUT BENCHMARK_HELPER write_worker if [ "$WARMUP_PASSES" -gt 0 ]; then @@ -614,17 +414,9 @@ EOF queue="$OUT_DIR/request-queue.args" write_queue "$queue" "$PASSES" measured_started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - measured_started_epoch="$(python3 - <<'PY' -import time -print(f"{time.time():.6f}") -PY -)" + measured_started_epoch="$("$BENCHMARK_HELPER" epoch)" xargs -0 -P "$CONCURRENCY" -n 6 /bin/bash "$OUT_DIR/run-one.sh" <"$queue" - measured_finished_epoch="$(python3 - <<'PY' -import time -print(f"{time.time():.6f}") -PY -)" + measured_finished_epoch="$("$BENCHMARK_HELPER" epoch)" measured_finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" if [ -n "${STATS_PID:-}" ]; then kill "$STATS_PID" >/dev/null 2>&1 || true @@ -640,20 +432,12 @@ PY cat "$line" >>"$OUT_DIR/requests.csv" done - python3 - "$OUT_DIR/run.json" "$measured_started_at" "$measured_finished_at" "$measured_started_epoch" "$measured_finished_epoch" <<'PY' -import json -import sys - -path, started_at, finished_at, started, finished = sys.argv[1:] -with open(path, encoding="utf-8") as fh: - run = json.load(fh) -run["measured_started_at"] = started_at -run["measured_finished_at"] = finished_at -run["measured_duration_seconds"] = round(float(finished) - float(started), 6) -with open(path, "w", encoding="utf-8") as fh: - json.dump(run, fh, separators=(",", ":")) - fh.write("\n") -PY + "$BENCHMARK_HELPER" update-run \ + "$OUT_DIR/run.json" \ + "$measured_started_at" \ + "$measured_finished_at" \ + "$measured_started_epoch" \ + "$measured_finished_epoch" python3 "$ROOT_DIR/scripts/benchmark-summary.py" "$OUT_DIR/requests.csv" "$OUT_DIR/summary.csv" python3 "$ROOT_DIR/scripts/benchmark-stats-summary.py" "$OUT_DIR/container-stats.jsonl" "$OUT_DIR/resource-summary.csv" diff --git a/scripts/conformance.sh b/scripts/conformance.sh index 595ad62..664cc9b 100755 --- a/scripts/conformance.sh +++ b/scripts/conformance.sh @@ -6,9 +6,9 @@ BASE_URL="${1:-${TRIPLET_BASE_URL:-http://localhost:8080}}" IDENTIFIER="${2:-${TRIPLET_CONFORMANCE_IDENTIFIER:-67352ccc-d1b0-11e1-89ae-279075081939.png}}" ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -TMPDIR="$(mktemp -d)" +CONFORMANCE_TMP_DIR="$(mktemp -d)" cleanup() { - rm -rf "$TMPDIR" + rm -rf "$CONFORMANCE_TMP_DIR" } trap cleanup EXIT @@ -42,47 +42,24 @@ assert_header_contains() { } echo "Checking base redirect" -curl -fsS -D "$TMPDIR/base.headers" -o /dev/null "$BASE_IMAGE_URL" -assert_header_contains "$TMPDIR/base.headers" "HTTP/1.1 303" -assert_header_contains "$TMPDIR/base.headers" "Location:" +curl -fsS -D "$CONFORMANCE_TMP_DIR/base.headers" -o /dev/null "$BASE_IMAGE_URL" +assert_header_contains "$CONFORMANCE_TMP_DIR/base.headers" "HTTP/1.1 303" +assert_header_contains "$CONFORMANCE_TMP_DIR/base.headers" "Location:" echo "Fetching $INFO_URL" -curl -fsS -H "Origin: https://viewer.example.edu" -D "$TMPDIR/info.headers" "$INFO_URL" -o "$TMPDIR/info.json" -assert_header_contains "$TMPDIR/info.headers" "Access-Control-Allow-Origin: *" -assert_header_contains "$TMPDIR/info.headers" "rel=\"profile\"" +curl -fsS -H "Origin: https://viewer.example.edu" -D "$CONFORMANCE_TMP_DIR/info.headers" "$INFO_URL" -o "$CONFORMANCE_TMP_DIR/info.json" +assert_header_contains "$CONFORMANCE_TMP_DIR/info.headers" "Access-Control-Allow-Origin: *" +assert_header_contains "$CONFORMANCE_TMP_DIR/info.headers" "rel=\"profile\"" echo "Checking info HEAD" -curl -fsSI "$INFO_URL" -o "$TMPDIR/info-head.headers" -assert_header_contains "$TMPDIR/info-head.headers" "HTTP/1.1 200" - -python3 - "$TMPDIR/info.json" <<'PY' -import json -import sys - -with open(sys.argv[1], "r", encoding="utf-8") as fh: - doc = json.load(fh) - -required = { - "@context": "http://iiif.io/api/image/3/context.json", - "type": "ImageService3", - "protocol": "http://iiif.io/api/image", - "profile": "level2", -} -for key, want in required.items(): - got = doc.get(key) - if got != want: - raise SystemExit(f"{key}: got {got!r}, want {want!r}") - -for key in ("id", "width", "height"): - if key not in doc: - raise SystemExit(f"missing {key}") -PY +curl -fsSI "$INFO_URL" -o "$CONFORMANCE_TMP_DIR/info-head.headers" +assert_header_contains "$CONFORMANCE_TMP_DIR/info-head.headers" "HTTP/1.1 200" if [ -n "$PRESENTATION_WRITE_TOKEN" ]; then WRITE_CANVAS="conformance-write-$$" WRITE_URL="${PRESENTATION_BASE_URL}/item-1/canvas/${WRITE_CANVAS}/annotations" - WRITE_BODY="${TMPDIR}/write-annotations.json" - UPDATED_BODY="${TMPDIR}/write-annotations-updated.json" + WRITE_BODY="${CONFORMANCE_TMP_DIR}/write-annotations.json" + UPDATED_BODY="${CONFORMANCE_TMP_DIR}/write-annotations-updated.json" cat > "$WRITE_BODY" <&2 exit 1 fi echo "Checking Presentation create precondition" - curl -fsS -X PUT -H "Authorization: Bearer ${PRESENTATION_WRITE_TOKEN}" -H "If-Match: *" -H "Content-Type: application/ld+json" -D "$TMPDIR/write-create.headers" --data-binary "@${WRITE_BODY}" "$WRITE_URL" -o /dev/null - WRITE_ETAG="$(awk 'tolower($0) ~ /^etag:/ {gsub(/\r/, "", $0); sub(/^[^:]+:[[:space:]]*/, "", $0); print; exit}' "$TMPDIR/write-create.headers")" + curl -fsS -X PUT -H "Authorization: Bearer ${PRESENTATION_WRITE_TOKEN}" -H "If-None-Match: *" -H "Content-Type: application/ld+json" -D "$CONFORMANCE_TMP_DIR/write-create.headers" --data-binary "@${WRITE_BODY}" "$WRITE_URL" -o /dev/null + assert_header_contains "$CONFORMANCE_TMP_DIR/write-create.headers" "HTTP/1.1 201" + WRITE_ETAG="$(awk 'tolower($0) ~ /^etag:/ {gsub(/\r/, "", $0); sub(/^[^:]+:[[:space:]]*/, "", $0); print; exit}' "$CONFORMANCE_TMP_DIR/write-create.headers")" if [ -z "$WRITE_ETAG" ]; then echo "annotation PUT create missing ETag" >&2 exit 1 fi echo "Checking Presentation update precondition" - curl -fsS -X PUT -H "Authorization: Bearer ${PRESENTATION_WRITE_TOKEN}" -H "If-Match: ${WRITE_ETAG}" -H "Content-Type: application/ld+json" -D "$TMPDIR/write-update.headers" --data-binary "@${UPDATED_BODY}" "$WRITE_URL" -o /dev/null + curl -fsS -X PUT -H "Authorization: Bearer ${PRESENTATION_WRITE_TOKEN}" -H "If-Match: ${WRITE_ETAG}" -H "Content-Type: application/ld+json" -D "$CONFORMANCE_TMP_DIR/write-update.headers" --data-binary "@${UPDATED_BODY}" "$WRITE_URL" -o /dev/null + UPDATED_ETAG="$(awk 'tolower($0) ~ /^etag:/ {gsub(/\r/, "", $0); sub(/^[^:]+:[[:space:]]*/, "", $0); print; exit}' "$CONFORMANCE_TMP_DIR/write-update.headers")" + if [ -z "$UPDATED_ETAG" ]; then + echo "annotation PUT update missing ETag" >&2 + exit 1 + fi got="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT -H "Authorization: Bearer ${PRESENTATION_WRITE_TOKEN}" -H "If-Match: ${WRITE_ETAG}" -H "Content-Type: application/ld+json" --data-binary "@${UPDATED_BODY}" "$WRITE_URL")" if [ "$got" != "412" ]; then echo "stale annotation PUT returned HTTP $got, want 412" >&2 exit 1 fi + + echo "Checking Presentation conditional delete" + curl -fsS -X DELETE -H "Authorization: Bearer ${PRESENTATION_WRITE_TOKEN}" -H "If-Match: ${UPDATED_ETAG}" "$WRITE_URL" -o /dev/null + assert_status 404 "$WRITE_URL" fi echo "Fetching $IMAGE_URL" -curl -fsS -D "$TMPDIR/default.headers" "$IMAGE_URL" -o "$TMPDIR/default.jpg" -test -s "$TMPDIR/default.jpg" -assert_header_contains "$TMPDIR/default.headers" "Content-Type: image/jpeg" -assert_header_contains "$TMPDIR/default.headers" "ETag:" -assert_header_contains "$TMPDIR/default.headers" "rel=\"canonical\"" +curl -fsS -D "$CONFORMANCE_TMP_DIR/default.headers" "$IMAGE_URL" -o "$CONFORMANCE_TMP_DIR/default.jpg" +test -s "$CONFORMANCE_TMP_DIR/default.jpg" +assert_header_contains "$CONFORMANCE_TMP_DIR/default.headers" "Content-Type: image/jpeg" +assert_header_contains "$CONFORMANCE_TMP_DIR/default.headers" "ETag:" +assert_header_contains "$CONFORMANCE_TMP_DIR/default.headers" "rel=\"canonical\"" -ETAG="$(awk 'tolower($0) ~ /^etag:/ {gsub(/\r/, "", $0); sub(/^[^:]+:[[:space:]]*/, "", $0); print; exit}' "$TMPDIR/default.headers")" +ETAG="$(awk 'tolower($0) ~ /^etag:/ {gsub(/\r/, "", $0); sub(/^[^:]+:[[:space:]]*/, "", $0); print; exit}' "$CONFORMANCE_TMP_DIR/default.headers")" if [ -z "$ETAG" ]; then echo "missing ETag value" >&2 exit 1 @@ -134,14 +121,14 @@ if [ "$got" != "304" ]; then fi echo "Checking PNG derivative" -curl -fsS -D "$TMPDIR/png.headers" "${BASE_IMAGE_URL}/full/32,/0/default.png" -o "$TMPDIR/default.png" -test -s "$TMPDIR/default.png" -assert_header_contains "$TMPDIR/png.headers" "Content-Type: image/png" +curl -fsS -D "$CONFORMANCE_TMP_DIR/png.headers" "${BASE_IMAGE_URL}/full/32,/0/default.png" -o "$CONFORMANCE_TMP_DIR/default.png" +test -s "$CONFORMANCE_TMP_DIR/default.png" +assert_header_contains "$CONFORMANCE_TMP_DIR/png.headers" "Content-Type: image/png" echo "Checking PDF derivative" -curl -fsS -D "$TMPDIR/pdf.headers" "${BASE_IMAGE_URL}/full/32,/0/default.pdf" -o "$TMPDIR/default.pdf" -test -s "$TMPDIR/default.pdf" -assert_header_contains "$TMPDIR/pdf.headers" "Content-Type: application/pdf" +curl -fsS -D "$CONFORMANCE_TMP_DIR/pdf.headers" "${BASE_IMAGE_URL}/full/32,/0/default.pdf" -o "$CONFORMANCE_TMP_DIR/default.pdf" +test -s "$CONFORMANCE_TMP_DIR/default.pdf" +assert_header_contains "$CONFORMANCE_TMP_DIR/pdf.headers" "Content-Type: application/pdf" echo "Checking syntax rejection" assert_status 400 "${BASE_IMAGE_URL}/full/max/0/bogus.jpg" @@ -154,97 +141,28 @@ SERVER="${SERVER%%/*}" "$VALIDATOR" --scheme "$SCHEME" -s "$SERVER" -p iiif/3 -i "$IDENTIFIER" --version=3.0 --level=2 --quiet echo "Fetching $MANIFEST_URL" -curl -fsS -H "Origin: https://viewer.example.edu" -D "$TMPDIR/manifest.headers" "$MANIFEST_URL" -o "$TMPDIR/manifest.json" -assert_header_contains "$TMPDIR/manifest.headers" "Access-Control-Allow-Origin: *" -assert_header_contains "$TMPDIR/manifest.headers" "Content-Type: application/ld+json" +curl -fsS -H "Origin: https://viewer.example.edu" -D "$CONFORMANCE_TMP_DIR/manifest.headers" "$MANIFEST_URL" -o "$CONFORMANCE_TMP_DIR/manifest.json" +assert_header_contains "$CONFORMANCE_TMP_DIR/manifest.headers" "Access-Control-Allow-Origin: *" +assert_header_contains "$CONFORMANCE_TMP_DIR/manifest.headers" "Content-Type: application/ld+json" echo "Checking manifest HEAD" -curl -fsSI "$MANIFEST_URL" -o "$TMPDIR/manifest-head.headers" -assert_header_contains "$TMPDIR/manifest-head.headers" "HTTP/1.1 200" +curl -fsSI "$MANIFEST_URL" -o "$CONFORMANCE_TMP_DIR/manifest-head.headers" +assert_header_contains "$CONFORMANCE_TMP_DIR/manifest-head.headers" "HTTP/1.1 200" echo "Fetching $ANNOTATIONS_URL" -curl -fsS -H "Origin: https://viewer.example.edu" -D "$TMPDIR/annotations.headers" "$ANNOTATIONS_URL" -o "$TMPDIR/annotations.json" -assert_header_contains "$TMPDIR/annotations.headers" "Access-Control-Allow-Origin: *" -assert_header_contains "$TMPDIR/annotations.headers" "Content-Type: application/ld+json" +curl -fsS -H "Origin: https://viewer.example.edu" -D "$CONFORMANCE_TMP_DIR/annotations.headers" "$ANNOTATIONS_URL" -o "$CONFORMANCE_TMP_DIR/annotations.json" +assert_header_contains "$CONFORMANCE_TMP_DIR/annotations.headers" "Access-Control-Allow-Origin: *" +assert_header_contains "$CONFORMANCE_TMP_DIR/annotations.headers" "Content-Type: application/ld+json" echo "Checking annotation page HEAD" -curl -fsSI "$ANNOTATIONS_URL" -o "$TMPDIR/annotations-head.headers" -assert_header_contains "$TMPDIR/annotations-head.headers" "HTTP/1.1 200" +curl -fsSI "$ANNOTATIONS_URL" -o "$CONFORMANCE_TMP_DIR/annotations-head.headers" +assert_header_contains "$CONFORMANCE_TMP_DIR/annotations-head.headers" "HTTP/1.1 200" echo "Validating Presentation API responses" -python3 - "$TMPDIR/manifest.json" "$TMPDIR/annotations.json" <<'PY' -import json -import re -import sys - -presentation_context = "http://iiif.io/api/presentation/3/context.json" -text_granularity_context = "http://iiif.io/api/extension/text-granularity/context.json" - - -def contexts(doc): - value = doc.get("@context") - if isinstance(value, str): - return {value} - if isinstance(value, list): - return {v for v in value if isinstance(v, str)} - return set() - - -with open(sys.argv[1], "r", encoding="utf-8") as fh: - manifest = json.load(fh) - -if presentation_context not in contexts(manifest): - raise SystemExit("manifest missing Presentation 3 context") -if manifest.get("type") != "Manifest": - raise SystemExit(f"manifest type = {manifest.get('type')!r}, want Manifest") -if not manifest.get("id"): - raise SystemExit("manifest id is required") -items = manifest.get("items") -if not isinstance(items, list) or not items: - raise SystemExit("manifest items must be a non-empty list") -canvas = items[0] -if canvas.get("type") != "Canvas": - raise SystemExit("manifest first item must be a Canvas") -if not canvas.get("id") or not isinstance(canvas.get("annotations"), list): - raise SystemExit("canvas id and annotations are required") - -with open(sys.argv[2], "r", encoding="utf-8") as fh: - page = json.load(fh) - -page_contexts = contexts(page) -if presentation_context not in page_contexts: - raise SystemExit("annotation page missing Presentation 3 context") -if text_granularity_context not in page_contexts: - raise SystemExit("annotation page missing Text Granularity context") -if page.get("type") != "AnnotationPage": - raise SystemExit(f"annotation page type = {page.get('type')!r}, want AnnotationPage") -annotations = page.get("items") -if not isinstance(annotations, list) or not annotations: - raise SystemExit("annotation page items must be a non-empty list") - -for idx, annotation in enumerate(annotations): - if annotation.get("type") != "Annotation": - raise SystemExit(f"annotation {idx} type must be Annotation") - if annotation.get("textGranularity") not in {"line", "word", "glyph"}: - raise SystemExit(f"annotation {idx} has invalid textGranularity") - motivation = annotation.get("motivation") - if isinstance(motivation, str): - motivations = {motivation} - elif isinstance(motivation, list): - motivations = {v for v in motivation if isinstance(v, str)} - else: - motivations = set() - if "supplementing" not in motivations: - raise SystemExit(f"annotation {idx} must include supplementing motivation") - body = annotation.get("body") - if not isinstance(body, dict) or body.get("type") != "TextualBody" or not isinstance(body.get("value"), str): - raise SystemExit(f"annotation {idx} must have a TextualBody value") - target = annotation.get("target") - if not isinstance(target, dict) or target.get("type") != "SpecificResource": - raise SystemExit(f"annotation {idx} target must be a SpecificResource") - selector = target.get("selector") - if not isinstance(selector, dict) or selector.get("type") != "FragmentSelector": - raise SystemExit(f"annotation {idx} selector must be a FragmentSelector") - if not re.fullmatch(r"xywh=\d+,\d+,\d+,\d+", str(selector.get("value", ""))): - raise SystemExit(f"annotation {idx} selector must be an xywh fragment") -PY +( + cd "$ROOT_DIR" + go run ./cmd/triplet-conformance-check \ + -info "$CONFORMANCE_TMP_DIR/info.json" \ + -manifest "$CONFORMANCE_TMP_DIR/manifest.json" \ + -annotation-page "$CONFORMANCE_TMP_DIR/annotations.json" +) diff --git a/scripts/ensure-iiif-validator.sh b/scripts/ensure-iiif-validator.sh index 864c0c1..b08f7e2 100755 --- a/scripts/ensure-iiif-validator.sh +++ b/scripts/ensure-iiif-validator.sh @@ -8,11 +8,10 @@ VALIDATOR="$VENV_DIR/bin/iiif-validate.py" PACKAGE_VERSION="${IIIF_VALIDATOR_VERSION:-1.0.5}" check_runtime() { - if "$VENV_DIR/bin/python" - <<'PY' >/dev/null 2>&1 -import magic -magic.from_buffer(b"IIIF validator libmagic check") -PY - then + # The installed entry point imports python-magic before it processes CLI + # arguments, so --help verifies both the Python package and native libmagic + # without embedding Python source in this shell script. + if "$VALIDATOR" --help >/dev/null 2>&1; then return 0 fi diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index 759c518..d971d9c 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -24,6 +24,8 @@ BRANCH="${GIT_BRANCH:-main}" TRIPLET_IMAGE="${TRIPLET_IMAGE:-ghcr.io/libops/triplet:${BRANCH}}" export TRIPLET_IMAGE export TRIPLET_CONFORMANCE_IDENTIFIER="$CONFORMANCE_IDENTIFIER" +export TRIPLET_PRESENTATION_WRITE_ENABLED="${TRIPLET_PRESENTATION_WRITE_ENABLED:-true}" +export TRIPLET_PRESENTATION_WRITE_TOKEN="${TRIPLET_PRESENTATION_WRITE_TOKEN:-triplet-integration-token}" sample_png_sha() { if command -v sha256sum >/dev/null 2>&1; then diff --git a/scripts/test.sh b/scripts/test.sh index 2086296..e1a6534 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -22,8 +22,8 @@ mkdir -p .cache/go-build .cache/go-mod # so integration tests run alongside unit tests. Otherwise they are skipped. NETWORK_ARGS=() ENV_ARGS=() -DB_USER="${MARIADB_USER:-scribe}" -DB_NAME="${MARIADB_DATABASE:-scribe}" +DB_USER="${MARIADB_USER:-triplet}" +DB_NAME="${MARIADB_DATABASE:-triplet}" MARIADB_ID=$(docker compose "${COMPOSE_ARGS[@]}" ps -q mariadb 2>/dev/null | head -1 || true) if [ -n "$MARIADB_ID" ]; then @@ -36,7 +36,7 @@ if [ -n "$MARIADB_ID" ]; then if [ -f "./secrets/mariadb_password" ]; then DB_PASSWORD=$(tr -d '\n' < ./secrets/mariadb_password) else - DB_PASSWORD="scribe" + DB_PASSWORD="triplet" fi ENV_ARGS+=(-e "TEST_DSN=${DB_USER}:${DB_PASSWORD}@tcp(mariadb:3306)/${DB_NAME}?parseTime=true") fi