Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 42 additions & 4 deletions .github/workflows/xtest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ on:
type: string
default: all
description: "SDK to focus on (go, js, java, all)"
otdfctl-source:
required: false
type: string
default: auto
description: "otdfctl source: 'auto' (detect from platform checkout), 'standalone', or 'platform'"
workflow_call:
inputs:
platform-ref:
Expand All @@ -50,6 +55,10 @@ on:
required: false
type: string
default: all
otdfctl-source:
required: false
type: string
default: auto
schedule:
- cron: "30 6 * * *" # 0630 UTC
- cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday)
Expand Down Expand Up @@ -198,8 +207,9 @@ jobs:
const tagToSha = {};
const headTags = [];

for (const { tag, head, sha, alias, err, release } of refInfo) {
const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? 'otdfctl' : sdkType == 'java' ? 'java-sdk' : sdkType)}`;
for (const { tag, head, sha, alias, err, release, source } of refInfo) {
const goRepoName = source === 'platform' ? 'platform' : 'otdfctl';
const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? goRepoName : sdkType == 'java' ? 'java-sdk' : sdkType)}`;
const sdkLink = `<a href="${htmlEscape(sdkRepoUrl)}">${htmlEscape(sdkType)}</a>`;
const commitLink = sha ? `<a href="${htmlEscape(`${sdkRepoUrl}/commit/${encodeURIComponent(sha)}`)}">${htmlEscape(sha.substring(0, 7))}</a>` : ' . ';
const tagLink = (release && tag)
Expand Down Expand Up @@ -290,6 +300,29 @@ jobs:
with:
node-version: "22.x"

######## DETECT PLATFORM-EMBEDDED OTDFCTL #############
- name: Detect platform-embedded otdfctl
id: detect-otdfctl
run: |
PLATFORM_DIR="${{ steps.run-platform.outputs.platform-working-dir }}"
if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then
if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then
echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl"
echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT"
echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT"
else
echo "otdfctl not found in platform checkout; using standalone repo"
echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT"
fi
elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then
echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT"
echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT"
else
echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT"
fi
env:
OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }}

######### CHECKOUT JS CLI #############
- name: Configure js-sdk
id: configure-js
Expand Down Expand Up @@ -324,6 +357,7 @@ jobs:
path: otdftests/xtest/sdk
sdk: go
version-info: "${{ needs.resolve-versions.outputs.go }}"
platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }}

- name: Cache Go modules
if: fromJson(steps.configure-go.outputs.heads)[0] != null
Expand All @@ -345,11 +379,15 @@ jobs:
OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }}

- name: Replace otdfctl go.mod packages, but only at head version of platform
if: fromJson(steps.configure-go.outputs.heads)[0] != null && env.FOCUS_SDK == 'go' && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
if: >-
steps.detect-otdfctl.outputs.otdfctl-source != 'platform'
&& fromJson(steps.configure-go.outputs.heads)[0] != null
&& env.FOCUS_SDK == 'go'
&& contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
env:
PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }}
run: |-
echo "Replacing go.mod packages..."
echo "Replacing go.mod packages (standalone otdfctl)..."
PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}"
OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/"
echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS"
Expand Down
6 changes: 5 additions & 1 deletion otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import json
import os
from typing import Annotated, Any, Optional

import typer
Expand Down Expand Up @@ -112,10 +113,13 @@ def resolve_versions(
raise typer.Exit(2)
infix = SDK_TAG_INFIXES.get(sdk)

# Allow overriding the Go SDK source (standalone otdfctl repo vs platform monorepo)
go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None

results: list[ResolveResult] = []
shas: set[str] = set()
for version in tags:
v = resolve(sdk, version, infix)
v = resolve(sdk, version, infix, go_source=go_source)
if is_resolve_success(v):
env = lookup_additional_options(sdk, v["tag"])
if env:
Expand Down
41 changes: 41 additions & 0 deletions otdf-sdk-mgr/src/otdf_sdk_mgr/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ def get_sdk_dirs() -> dict[str, Path]:
}

GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl"
GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl"

GO_MODULE_PATH = "github.com/opentdf/otdfctl"
GO_MODULE_PATH_PLATFORM = "github.com/opentdf/platform/otdfctl"

LTS_VERSIONS: dict[str, str] = {
"go": "0.24.0",
Expand Down Expand Up @@ -111,4 +115,41 @@ def get_sdk_dirs() -> dict[str, Path]:
"platform": "service",
}

# When resolving go versions from the platform repo, use "otdfctl" infix
# (tags are otdfctl/v0.X.Y in the platform monorepo)
SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl"


def go_git_url(source: str | None = None) -> str:
"""Return the git URL for Go SDK resolution based on source.

Args:
source: "platform" to use the platform monorepo, None/"standalone" for the
standalone otdfctl repo.
"""
if source == "platform":
return SDK_GIT_URLS["platform"]
return SDK_GIT_URLS["go"]


def go_tag_infix(source: str | None = None) -> str | None:
"""Return the tag infix for Go SDK resolution based on source."""
if source == "platform":
return SDK_TAG_INFIXES_PLATFORM_GO
return None


def go_install_prefix(source: str | None = None) -> str:
"""Return the go install/run prefix based on source."""
if source == "platform":
return GO_INSTALL_PREFIX_PLATFORM
return GO_INSTALL_PREFIX


def go_module_path(source: str | None = None) -> str:
"""Return the Go module path based on source."""
if source == "platform":
return GO_MODULE_PATH_PLATFORM
return GO_MODULE_PATH

ALL_SDKS = ["go", "js", "java"]
20 changes: 15 additions & 5 deletions otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@
from pathlib import Path

from otdf_sdk_mgr.config import (
GO_MODULE_PATH_PLATFORM,
LTS_VERSIONS,
get_sdk_dir,
get_sdk_dirs,
go_install_prefix,
)
from otdf_sdk_mgr.checkout import checkout_sdk_branch
from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions
Expand All @@ -24,22 +26,30 @@ class InstallError(Exception):
"""Raised when SDK installation fails."""


def install_go_release(version: str, dist_dir: Path) -> None:
def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None:
"""Install a Go CLI release by writing a .version file.

The cli.sh and otdfctl.sh wrappers read .version and use
`go run github.com/opentdf/otdfctl@{version}` instead of a local binary.
The cli.sh and otdfctl.sh wrappers read .version (and optionally .module-path)
and use `go run <module>@{version}` instead of a local binary.

Args:
version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0").
dist_dir: Target distribution directory.
source: "platform" to use the platform monorepo module path, None for standalone.
"""
go_dir = get_sdk_dir() / "go"
dist_dir.mkdir(parents=True, exist_ok=True)
tag = normalize_version(version)
(dist_dir / ".version").write_text(f"{tag}\n")
if source == "platform":
(dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n")
Comment on lines +44 to +45
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The source parameter is currently not being propagated from the CLI or the install_release function. Consequently, the .module-path file will never be created, causing the Go wrappers to use the incorrect module path for platform-embedded releases.

You can fix this by inferring the source from the version prefix if it is not explicitly provided.

Suggested change
if source == "platform":
(dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n")
if source == "platform" or version.startswith("otdfctl/"):
(dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n")

shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh")
shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh")
shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml")
print(f" Pre-warming Go cache for otdfctl@{tag}...")
install_module = go_install_prefix(source).removeprefix("go run ")
print(f" Pre-warming Go cache for {install_module}@{tag}...")
result = subprocess.run(
["go", "install", f"github.com/opentdf/otdfctl@{tag}"],
["go", "install", f"{install_module}@{tag}"],
capture_output=True,
text=True,
)
Expand Down
Loading
Loading