Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/scheduled-smoke-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ jobs:
- name: Verify SDK Generation
run: |
Write-Host "Checking SDK generation results..."
$sdkPath = "${{ github.workspace }}/eng/sdkGen/csharp/src/EdFi.DmsApi.TestSdk/bin/Release/net8.0/EdFi.DmsApi.TestSdk.dll"
$sdkPath = "${{ github.workspace }}/eng/sdkGen/csharp/src/EdFi.DmsApi.TestSdk/bin/Release/net10.0/EdFi.DmsApi.TestSdk.dll"
Write-Host "Expected SDK path: $sdkPath"

if (Test-Path $sdkPath) {
Expand Down Expand Up @@ -158,12 +158,12 @@ jobs:

- name: Run NonDestructive DMS SDK Tests
run: |
./Invoke-NonDestructiveSdkTests.ps1 -BaseUrl "http://localhost:8080" -Key "$env:SMOKE_TEST_KEY" -Secret "$env:SMOKE_TEST_SECRET" -SdkPath "${{ github.workspace }}/eng/sdkGen/csharp/src/EdFi.DmsApi.TestSdk/bin/Release/net8.0/EdFi.DmsApi.TestSdk.dll"
./Invoke-NonDestructiveSdkTests.ps1 -BaseUrl "http://localhost:8080" -Key "$env:SMOKE_TEST_KEY" -Secret "$env:SMOKE_TEST_SECRET" -SdkPath "${{ github.workspace }}/eng/sdkGen/csharp/src/EdFi.DmsApi.TestSdk/bin/Release/net10.0/EdFi.DmsApi.TestSdk.dll"
working-directory: eng/smoke_test/

- name: Run Destructive DMS SDK Tests
run: |
./Invoke-DestructiveSdkTests.ps1 -BaseUrl "http://localhost:8080" -Key "$env:SMOKE_TEST_KEY" -Secret "$env:SMOKE_TEST_SECRET" -SdkPath "${{ github.workspace }}/eng/sdkGen/csharp/src/EdFi.DmsApi.TestSdk/bin/Release/net8.0/EdFi.DmsApi.TestSdk.dll"
./Invoke-DestructiveSdkTests.ps1 -BaseUrl "http://localhost:8080" -Key "$env:SMOKE_TEST_KEY" -Secret "$env:SMOKE_TEST_SECRET" -SdkPath "${{ github.workspace }}/eng/sdkGen/csharp/src/EdFi.DmsApi.TestSdk/bin/Release/net10.0/EdFi.DmsApi.TestSdk.dll"
working-directory: eng/smoke_test/

- name: Run NonDestructive ODS SDK Tests
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ CLAUDE.local.md

# SDK generation artifacts
/eng/sdkGen/csharp/*
openApi-codegen-cli.jar
openApi-codegen-cli-*.jar
/.github/copilot-instructions.md

# Claude temporary files leaking into the project root in 2.1.5
Expand Down
198 changes: 154 additions & 44 deletions build-sdk.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -77,69 +77,177 @@

Import-Module -Name "$PSScriptRoot/eng/build-helpers.psm1" -Force

$openApiGeneratorVersion = "7.19.0"
$openApiGeneratorJar = "openApi-codegen-cli-$openApiGeneratorVersion.jar"

$solutionRoot = "$PSScriptRoot/$OutputFolder"
$projectPath = "$solutionRoot/src/$PackageName/$PackageName.csproj"
$nuspecPath = "$PSScriptRoot/eng/sdkGen/$PackageName.nuspec"

function DownloadCodeGen {
if (-not (Test-Path -Path openApi-codegen-cli.jar)) {
Invoke-WebRequest -OutFile openApi-codegen-cli.jar https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.9.0/openapi-generator-cli-7.9.0.jar
if (-not (Test-Path -Path $openApiGeneratorJar)) {
Invoke-WebRequest -OutFile $openApiGeneratorJar https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/$openApiGeneratorVersion/openapi-generator-cli-$openApiGeneratorVersion.jar
}
}

function GenerateSdk {
param (
[string]
$ApiPackage,
function Rename-DescriptorOperationId {
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Workaround. MetaEd.js emits operationIds like getGradingPeriods on both /ed-fi/gradingPeriods (resources-spec) and /ed-fi/gradingPeriodDescriptors (descriptors-spec). Once merged into a single spec, duplicate operationIds produce duplicate C# method names in Apis.All.

Removable when reference/design/backend-redesign/epics/08-relational-read-path/07-disambiguate-descriptor-operationids.md lands. Once descriptor operationIds carry the Descriptor/Descriptors suffix in MetaEd.js output, this helper plus the descriptor loop in Merge-DmsSpecs (lines 127-133) become dead code.

param([string]$old)
if ($old -match 'ById$') {
return ($old -replace 'ById$', 'DescriptorById')
} elseif ($old -match 's$') {
return ($old -replace 's$', 'Descriptors')
} else {
return "${old}Descriptor"
}
}

function Merge-DmsSpecs {

Check warning

Code scanning / PSScriptAnalyzer

Cmdlet Singular Noun Warning

The cmdlet 'Merge-DmsSpecs' uses a plural noun. A singular noun should be used instead.
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Permanent improvement. The two-pass flow (one GenerateSdk per spec) used to let the descriptors pass clobber Client/HostConfiguration.cs written by the resources pass, leaving every resource-side IxxxApi DI registration absent (232 descriptor registrations only, vs the expected 396). Merging into one OpenAPI document and generating once eliminates the clobber as a structural class of bug; this is the correct shape independent of any upstream change.

# Merges resources-spec and descriptors-spec into a single OpenAPI document so the generator
# runs once and emits a complete HostConfiguration.cs / IApi.cs. The two-pass flow used to
# let the descriptors pass clobber the resources-pass HostConfiguration, leaving resource
# APIs unregistered in DI and the smoke test tool throwing NREs on every resource call.
param(
[string]$ResourcesEndpoint,
[string]$DescriptorsEndpoint
)

$resources = (Invoke-WebRequest -Uri $ResourcesEndpoint).Content | ConvertFrom-Json -Depth 100 -AsHashtable
$descriptors = (Invoke-WebRequest -Uri $DescriptorsEndpoint).Content | ConvertFrom-Json -Depth 100 -AsHashtable

# Rename descriptor operationIds that collide with resource ones (e.g. getGradingPeriods
# appears in both specs; without renaming we'd get duplicate C# method names in Apis.All).
$resourceIds = @{}
foreach ($pathOps in $resources.paths.Values) {
foreach ($op in $pathOps.Values) {
if ($op -is [System.Collections.IDictionary] -and $op.ContainsKey('operationId')) {
$resourceIds[$op.operationId] = $true
}
}
}
foreach ($pathOps in $descriptors.paths.Values) {
foreach ($op in $pathOps.Values) {
if ($op -is [System.Collections.IDictionary] -and $op.ContainsKey('operationId') -and $resourceIds.ContainsKey($op.operationId)) {
$op.operationId = Rename-DescriptorOperationId $op.operationId
}
}
}

# Union descriptor paths/schemas/tags into resources. Parameters/responses/securitySchemes
# are identical across both specs (verified by inspection) so the resources copy is kept.
foreach ($pathName in $descriptors.paths.Keys) {
if (-not $resources.paths.ContainsKey($pathName)) {
$resources.paths[$pathName] = $descriptors.paths[$pathName]
}
}
foreach ($schemaName in $descriptors.components.schemas.Keys) {
if (-not $resources.components.schemas.ContainsKey($schemaName)) {
$resources.components.schemas[$schemaName] = $descriptors.components.schemas[$schemaName]
}
}
$existingTagNames = @{}
foreach ($t in $resources.tags) { $existingTagNames[$t.name] = $true }
foreach ($t in $descriptors.tags) {
if (-not $existingTagNames.ContainsKey($t.name)) {
$resources.tags += $t
}
}

[string]
$ModelPackage,
# Drop required arrays on Homograph* schemas so the generichost-library generator omits its
# throw-on-missing-required validation. The smoke test tool's data factory cannot populate
# required props on these extension models, and the server-side spec still enforces them.
foreach ($schemaName in @($resources.components.schemas.Keys)) {
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Workaround. OpenAPI Generator 7.19.0's generichost library validates required properties at deserialization, emitting throw new ArgumentException("Property is required for class HomographXxx", ...) blocks the smoke test tool's reflection-based data factory can't satisfy. Affected models: HomographContact, HomographSchool, HomographStaff, HomographStudent, HomographSchoolReference. Without this strip, the SDK throws at app setup before any test runs.

Removable when either (a) the generichost template adds an opt-out for the throw-on-missing-required validation, or (b) EdFi.LoadTools.SmokeTest updates its data factory to populate required constructor params on extension models. No upstream story yet tracks this; safe in the interim because the server-side spec still enforces the requirement (validation just shifts from client throw to server 400).

if ($schemaName.StartsWith('Homograph')) {
$schema = $resources.components.schemas[$schemaName]
if ($schema -is [System.Collections.IDictionary] -and $schema.ContainsKey('required')) {
$schema.Remove('required')
}
}
}

return $resources
}

[string]
$Endpoint
function GenerateSdk {
param (
[string]$ApiPackage,
[string]$ModelPackage,
[System.Collections.IDictionary]$Spec
)

# Download and parse OpenAPI spec
$spec = Invoke-WebRequest -Uri $Endpoint | ConvertFrom-Json
# Rewrite tags on non-ed-fi paths whose tag also appears on /ed-fi/* paths. Without this,
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Workaround. MetaEd.js emits tag: contacts for both /ed-fi/contacts and /homograph/contacts even though it namespaces the URL path, operationId, and schema names. Without this rewrite, the generator emits a single ContactsApi class containing two Post methods, and the smoke test tool's categorizer fails with Multiple matching Post methods were found on type 'ContactsApi' before any test runs.

Removable when docs/stories/metaed-extension-openapi-tag-prefix.md lands and DMS picks up the new ApiSchema package. Once MetaEd.js emits tag: homograph_contacts directly, the coreTags collection and both foreach loops here (~20 lines) become dead code.

# /ed-fi/contacts and /homograph/contacts share tag 'contacts' and land on the same
# ContactsApi, which the smoke test tool's "one Post per Api class" categorizer can't
# disambiguate. The generator emits a distinct *Api class per tag, so prefixing the tag
# with the namespace splits the colliding endpoints into separate classes.
$coreTags = @{}
foreach ($pathName in @($Spec.paths.Keys)) {
if ($pathName.StartsWith('/ed-fi/')) {
foreach ($verb in @($Spec.paths[$pathName].Keys)) {
$op = $Spec.paths[$pathName][$verb]
if ($op -is [System.Collections.IDictionary] -and $op.ContainsKey('tags') -and $op['tags']) {
foreach ($t in $op['tags']) { $coreTags[$t] = $true }
}
}
}
}
foreach ($pathName in @($Spec.paths.Keys)) {
if ($pathName -match '^/(?<ns>[^/]+)/' -and $matches.ns -ne 'ed-fi') {
$nsSafe = ($matches.ns -replace '[^A-Za-z0-9]', '')
foreach ($verb in @($Spec.paths[$pathName].Keys)) {
$op = $Spec.paths[$pathName][$verb]
if ($op -is [System.Collections.IDictionary] -and $op.ContainsKey('tags') -and $op['tags']) {
$op['tags'] = @($op['tags'] | ForEach-Object { if ($coreTags.ContainsKey($_)) { "${nsSafe}_$_" } else { $_ } })
}
}
}
}

# Find all operationIds that contain an underscore
$operationIds = $spec.paths.PSObject.Properties.Value | ForEach-Object {
$_.PSObject.Properties.Value | Where-Object { $_.operationId -and $_.operationId -like "*_*" } | ForEach-Object { $_.operationId }
# Build --operation-id-name-mappings for underscore-bearing operationIds so the generator
# preserves the namespace separator in C# method names (post_HomographContact stays
# Post_HomographContact rather than collapsing to PostHomographContact).
$operationIds = @()
foreach ($pathOps in $Spec.paths.Values) {
foreach ($op in $pathOps.Values) {
if ($op -is [System.Collections.IDictionary] -and $op.ContainsKey('operationId') -and $op.operationId -like "*_*") {
$operationIds += $op.operationId
}
}
}

# Normalize operationId to camelCase without underscores (for the left side of mapping)
function Normalize-OperationId {
param($opId)
$parts = $opId -split '_'
$camel = $parts[0] + ($parts[1..($parts.Count-1)] | ForEach-Object { $_.Substring(0,1).ToUpper() + $_.Substring(1) } | ForEach-Object { $_ }) -join ''
$camel = $parts[0] + (($parts[1..($parts.Count-1)] | ForEach-Object { $_.Substring(0,1).ToUpper() + $_.Substring(1) }) -join '')
return $camel
}

# Capitalize the first character of the string
function Capitalize-FirstChar {
param($s)
if ($s.Length -eq 0) { return $s }
return $s.Substring(0,1).ToUpper() + $s.Substring(1)
}

# Build mappings string: left = normalized, right = original with first char uppercased
$mappings = ($operationIds | Sort-Object -Unique | ForEach-Object { "$(Normalize-OperationId $_)=$(Capitalize-FirstChar $_)" }) -join ","
# Example --operation-id-name-mappings deleteHomographContactsById=Delete_HomographContactsById

& java -Xmx5g -jar openApi-codegen-cli.jar generate `
-g csharp `
-i $Endpoint `
--api-package $ApiPackage `
--model-package $ModelPackage `
-o $OutputFolder `
--operation-id-name-mappings $mappings `
--additional-properties "packageName=$PackageName,targetFramework=net8.0,netCoreProjectFile=true" `
--global-property modelTests=false `
--global-property apiTests=false `
--global-property apiDocs=false `
--global-property modelDocs=false `
--skip-validate-spec

$specTempPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "openapi-spec-$([System.Guid]::NewGuid()).json")
$Spec | ConvertTo-Json -Depth 100 -Compress | Set-Content -Path $specTempPath -Encoding UTF8NoBOM

try {
& java -Xmx5g -jar $openApiGeneratorJar generate `
-g csharp `
-i $specTempPath `
--api-package $ApiPackage `
--model-package $ModelPackage `
-o $OutputFolder `
--operation-id-name-mappings $mappings `
--additional-properties "packageName=$PackageName,targetFramework=net10.0,netCoreProjectFile=true" `
--global-property modelTests=false `
--global-property apiTests=false `
--global-property apiDocs=false `
--global-property modelDocs=false `
--skip-validate-spec
}
finally {
Remove-Item $specTempPath -ErrorAction SilentlyContinue
}
}

function BuildSdk {
Expand Down Expand Up @@ -198,16 +306,18 @@
function Invoke-BuildAndGenerateSdk {
Invoke-Step { DownloadCodeGen }

if ($PackageName -eq "EdFi.DmsApi.TestSdk") {
Invoke-Step { GenerateSdk -ApiPackage "Apis.All" -ModelPackage "Models.All" -Endpoint "$DmsUrl/metadata/specifications/resources-spec.json" }
Invoke-Step { GenerateSdk -ApiPackage "Apis.All" -ModelPackage "Models.All" -Endpoint "$DmsUrl/metadata/specifications/descriptors-spec.json" }
} elseif ($PackageName -eq "EdFi.DmsApi.Sdk") {
Invoke-Step { GenerateSdk -ApiPackage "Apis.Ed_Fi" -ModelPackage "Models.Ed_Fi" -Endpoint "$DmsUrl/metadata/specifications/resources-spec.json" }
Invoke-Step { GenerateSdk -ApiPackage "Apis.Ed_Fi" -ModelPackage "Models.Ed_Fi" -Endpoint "$DmsUrl/metadata/specifications/descriptors-spec.json" }
} else {
throw "Unknown PackageName value: $PackageName"
$mergedSpec = Merge-DmsSpecs `
-ResourcesEndpoint "$DmsUrl/metadata/specifications/resources-spec.json" `
-DescriptorsEndpoint "$DmsUrl/metadata/specifications/descriptors-spec.json"

$packagePair = switch ($PackageName) {
"EdFi.DmsApi.TestSdk" { @{ Api = "Apis.All"; Model = "Models.All" } }
"EdFi.DmsApi.Sdk" { @{ Api = "Apis.Ed_Fi"; Model = "Models.Ed_Fi" } }
default { throw "Unknown PackageName value: $PackageName" }
}

Invoke-Step { GenerateSdk -ApiPackage $packagePair.Api -ModelPackage $packagePair.Model -Spec $mergedSpec }

Invoke-Step { BuildSdk }
}

Expand Down
12 changes: 10 additions & 2 deletions eng/Package-Management.psm1
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,16 @@ function Get-NugetPackage {
$PreRelease
)

$lowerId = $PackageName.ToLower()
$packagesDir = ".packages"

if (-not [string]::IsNullOrWhiteSpace($PackageVersion) -and $PackageVersion.Split('.').Length -ge 3) {
$cachedPackage = Join-Path -Path $packagesDir -ChildPath "$lowerId.$PackageVersion"
if (Test-Path -Path $cachedPackage) {
return $cachedPackage
}
}

# Pre-releases
$nugetServicesURL = $ReleaseServiceIndex
if ($PreRelease) {
Expand All @@ -136,7 +146,6 @@ function Get-NugetPackage {
| Where-Object { $_."@type" -like "PackageBaseAddress*" } `
| Select-Object -Property "@id" -ExpandProperty "@id"

$lowerId = $PackageName.ToLower()
# Lookup available packages
$package = Invoke-RestMethod "$($packageService)$($lowerId)/index.json"
# Sort by SemVer
Expand Down Expand Up @@ -165,7 +174,6 @@ function Get-NugetPackage {

$file = "$($lowerId).$($version)"
$zip = "$($file).zip"
$packagesDir = ".packages"
New-Item -Path $packagesDir -Force -ItemType Directory | Out-Null

Push-Location $packagesDir
Expand Down
12 changes: 6 additions & 6 deletions eng/sdkGen/EdFi.DmsApi.Sdk.nuspec
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@
<copyright>Copyright @ $year$ Ed-Fi Alliance, LLC and Contributors</copyright>
<tags>Ed-Fi Data Management Service SDK</tags>
<dependencies>
<group targetFramework="net8.0">
<dependency id="NewtonSoft.Json" version="13.0.3" />
<dependency id="JsonSubTypes" version="2.0.1" />
<dependency id="RestSharp" version="112.0.0" />
<dependency id="Polly" version="8.1.0" />
<group targetFramework="net10.0">
<dependency id="Microsoft.Extensions.Http" version="10.0.1" />
<dependency id="Microsoft.Extensions.Hosting" version="10.0.1" />
<dependency id="Microsoft.Extensions.Http.Polly" version="10.0.1" />
<dependency id="Microsoft.Net.Http.Headers" version="10.0.1" />
</group>
</dependencies>
</metadata>
<files>
<file src="readme.txt" target="" />
<file src="csharp\src\EdFi.DmsApi.Sdk\bin\Release\net8.0\EdFi.DmsApi.Sdk.dll" target="lib\net8.0" />
<file src="csharp\src\EdFi.DmsApi.Sdk\bin\Release\net10.0\EdFi.DmsApi.Sdk.dll" target="lib\net10.0" />
</files>
</package>
12 changes: 6 additions & 6 deletions eng/sdkGen/EdFi.DmsApi.TestSdk.nuspec
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@
<copyright>Copyright @ $year$ Ed-Fi Alliance, LLC and Contributors</copyright>
<tags>Ed-Fi Data Management Service TestSdk</tags>
<dependencies>
<group targetFramework="net8.0">
<dependency id="NewtonSoft.Json" version="13.0.3" />
<dependency id="JsonSubTypes" version="2.0.1" />
<dependency id="RestSharp" version="112.0.0" />
<dependency id="Polly" version="8.1.0" />
<group targetFramework="net10.0">
<dependency id="Microsoft.Extensions.Http" version="10.0.1" />
<dependency id="Microsoft.Extensions.Hosting" version="10.0.1" />
<dependency id="Microsoft.Extensions.Http.Polly" version="10.0.1" />
<dependency id="Microsoft.Net.Http.Headers" version="10.0.1" />
</group>
</dependencies>
</metadata>
<files>
<file src="readme.txt" target="" />
<file src="csharp\src\EdFi.DmsApi.TestSdk\bin\Release\net8.0\EdFi.DmsApi.TestSdk.dll" target="lib\net8.0" />
<file src="csharp\src\EdFi.DmsApi.TestSdk\bin\Release\net10.0\EdFi.DmsApi.TestSdk.dll" target="lib\net10.0" />
</files>
</package>
2 changes: 1 addition & 1 deletion eng/smoke_test/Invoke-DestructiveSdkTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ if ($SdkPath) {
$sdkDllPath = Get-ApiSdkDll
}

$path = Get-SmokeTestTool -PackageVersion '7.3.10008' -PreRelease
$path = Get-SmokeTestTool -PackageVersion '7.3.20144' -PreRelease

$parameters = @{
BaseUrl = $BaseUrl
Expand Down
2 changes: 1 addition & 1 deletion eng/smoke_test/Invoke-NonDestructiveApiTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ $ErrorActionPreference = "Stop"
Import-Module ../Package-Management.psm1 -Force
Import-Module ./modules/SmokeTest.psm1

$path = Get-SmokeTestTool -PackageVersion '7.2.413'
$path = Get-SmokeTestTool -PackageVersion '7.3.20144' -PreRelease

$parameters = @{
BaseUrl = $BaseUrl
Expand Down
2 changes: 1 addition & 1 deletion eng/smoke_test/Invoke-NonDestructiveSdkTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ if ($SdkPath) {
$sdkDllPath = Get-ApiSdkDll
}

$path = Get-SmokeTestTool -PackageVersion '7.3.10008' -PreRelease
$path = Get-SmokeTestTool -PackageVersion '7.3.20144' -PreRelease

$parameters = @{
BaseUrl = $BaseUrl
Expand Down
Loading
Loading