diff --git a/data/payload.go b/data/payload.go index dfe188d..a0ed7ec 100644 --- a/data/payload.go +++ b/data/payload.go @@ -62,6 +62,16 @@ type payloadCache struct { documentation []DocumentationFile documentationErr error documentationLoaded bool + // refLicenses caches LicenseAtRef lookups by ref. More than one control + // evaluates the latest release's license, so without this the same + // endpoint would be hit once per control. + refLicenses map[string]refLicenseCacheEntry +} + +type refLicenseCacheEntry struct { + license RefLicense + found bool + err error } // AddEvidence, GetEvidence, and ClearEvidence implement gemara.HasEvidence. @@ -275,3 +285,30 @@ func (p *Payload) GetDocumentationFiles() ([]DocumentationFile, error) { p.cache.documentationLoaded = true return files, err } + +// GetLicenseAtRef returns the license GitHub detects at the given git ref, +// fetched once per payload per ref. Errors are cached deliberately, like +// GetDocumentationFiles: more than one control reads the same ref, and a +// lookup that just failed should surface the same evidence to all of them +// rather than being retried within one run. A payload without a cache (as +// built directly in tests) simply calls through uncached. +func (p *Payload) GetLicenseAtRef(ref string) (RefLicense, bool, error) { + if p.RestData == nil { + return RefLicense{}, false, fmt.Errorf("payload missing required repository data") + } + if p.cache != nil { + if p.cache.refLicenses != nil { + if entry, ok := p.cache.refLicenses[ref]; ok { + return entry.license, entry.found, entry.err + } + } + } + license, found, err := p.LicenseAtRef(ref) + if p.cache != nil { + if p.cache.refLicenses == nil { + p.cache.refLicenses = make(map[string]refLicenseCacheEntry) + } + p.cache.refLicenses[ref] = refLicenseCacheEntry{license: license, found: found, err: err} + } + return license, found, err +} diff --git a/data/rest-data.go b/data/rest-data.go index 6abcba6..61d3226 100644 --- a/data/rest-data.go +++ b/data/rest-data.go @@ -3,10 +3,12 @@ package data import ( "context" "encoding/json" + "errors" "fmt" "io" "log" "net/http" + "net/url" "strings" "sync" @@ -54,12 +56,20 @@ type RepoContent struct { } type ReleaseData struct { - Id int `json:"id"` - Name string `json:"name"` - TagName string `json:"tag_name"` - URL string `json:"url"` - Draft bool `json:"draft"` - Assets []ReleaseAsset `json:"assets"` + Id int `json:"id"` + Name string `json:"name"` + TagName string `json:"tag_name"` + URL string `json:"url"` + Draft bool `json:"draft"` + // Prerelease is GitHub's own flag for a release marked as not + // production-ready; it must be excluded from "latest published release" + // selection alongside drafts. + Prerelease bool `json:"prerelease"` + // PublishedAt is an RFC3339 timestamp. The /releases listing endpoint is + // ordered by creation time, not publish time, so selecting "the latest" + // release requires comparing this field rather than trusting list order. + PublishedAt string `json:"published_at"` + Assets []ReleaseAsset `json:"assets"` } type ReleaseAsset struct { @@ -523,6 +533,145 @@ func (r *RestData) getReleases() error { } } +// RefLicense describes the license GitHub detects in the repository tree at a +// specific git ref. +type RefLicense struct { + // SpdxId is GitHub's classification of the license file, or "NOASSERTION" + // when a license file exists but could not be identified. + SpdxId string + // Path is the location of the license file within the tree at the ref. + Path string +} + +// ErrRefUnresolvable is returned by RefExists when GitHub reports no commit +// at the given ref (a deleted or re-pointed tag). Distinguishing this from +// "ref exists but has no license file" matters because a 404 from the +// license-at-ref endpoint means the same thing in both cases. +var ErrRefUnresolvable = errors.New("ref does not resolve to a commit") + +// ErrRateLimited is returned when GitHub responds 403, which the retry layer +// treats as permanent rather than transient. A caller basing a verdict on a +// missing signal should treat this differently from "the signal is absent", +// since a rate-limited scan cannot observe the signal at all. +var ErrRateLimited = errors.New("request was rate limited (403)") + +// RefExists reports whether ref resolves to a commit. It is used to +// disambiguate a 404 from the license-at-ref endpoint: GitHub returns the same +// 404 whether the ref has no recognized license file or the ref itself no +// longer resolves (e.g. a tag deleted or force-moved after a release was +// published), and only the latter should be reported as ambiguous rather than +// "no license found". +func (r *RestData) RefExists(ref string) (bool, error) { + var logger hclog.Logger + if r.Config != nil { + logger = r.Config.Logger + } + if logger == nil { + logger = hclog.NewNullLogger() + } + if r.HttpClient == nil { + r.HttpClient = &http.Client{} + } + endpoint := fmt.Sprintf("%s/repos/%s/%s/commits/%s", APIBase, r.owner, r.repo, url.QueryEscape(ref)) + var exists bool + err := withRetry(logger, fmt.Sprintf("GET %s", endpoint), func() error { + request, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+r.token) + response, err := r.HttpClient.Do(request) + if err != nil { + return fmt.Errorf("error making http call: %s", err.Error()) + } + defer func() { _ = response.Body.Close() }() + switch { + case response.StatusCode == http.StatusNotFound: + exists = false + return nil + case response.StatusCode == http.StatusForbidden: + return ErrRateLimited + case response.StatusCode != http.StatusOK: + return fmt.Errorf("unexpected response: %s", response.Status) + default: + exists = true + return nil + } + }) + if err != nil { + return false, err + } + return exists, nil +} + +// LicenseAtRef fetches the license GitHub detects for the repository at the +// given git ref (typically a release tag). GitHub's auto-generated release +// archives contain the tree at the tag, so this observes the license actually +// shipped with a release, which the default-branch license may no longer match. +// +// A 404 means GitHub found no license file at that ref; it is reported as +// found=false with a nil error, distinct from request failures. MakeApiCall is +// not used here because its uniform non-200 error cannot make that distinction. +// A 403 is reported as ErrRateLimited rather than a generic error, since a +// caller basing a verdict on this lookup's absence needs to tell "checked and +// found nothing" apart from "could not check". +func (r *RestData) LicenseAtRef(ref string) (license RefLicense, found bool, err error) { + var logger hclog.Logger + if r.Config != nil { + logger = r.Config.Logger + } + if logger == nil { + logger = hclog.NewNullLogger() + } + if r.HttpClient == nil { + r.HttpClient = &http.Client{} + } + endpoint := fmt.Sprintf("%s/repos/%s/%s/license?ref=%s", APIBase, r.owner, r.repo, url.QueryEscape(ref)) + err = withRetry(logger, fmt.Sprintf("GET %s", endpoint), func() error { + request, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+r.token) + response, err := r.HttpClient.Do(request) + if err != nil { + return fmt.Errorf("error making http call: %s", err.Error()) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode == http.StatusNotFound { + // No license file exists at this ref. That is an observation about + // the release, not a request failure. + return nil + } + if response.StatusCode == http.StatusForbidden { + return ErrRateLimited + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected response: %s", response.Status) + } + body, err := io.ReadAll(response.Body) + if err != nil { + return err + } + var decoded struct { + Path string `json:"path"` + License struct { + SpdxId string `json:"spdx_id"` + } `json:"license"` + } + if err := json.Unmarshal(body, &decoded); err != nil { + return fmt.Errorf("failed to decode license data for ref %q: %w", ref, err) + } + license = RefLicense{SpdxId: decoded.License.SpdxId, Path: decoded.Path} + found = true + return nil + }) + if err != nil { + return RefLicense{}, false, err + } + return license, found, nil +} + func (r *RestData) getWorkflowPermissions() error { endpoint := fmt.Sprintf("%s/repos/%s/%s/actions/permissions", APIBase, r.owner, r.repo) responseData, err := r.MakeApiCall(endpoint, true) diff --git a/data/rest-data_test.go b/data/rest-data_test.go index 41df6ae..7f1088c 100644 --- a/data/rest-data_test.go +++ b/data/rest-data_test.go @@ -579,3 +579,177 @@ func TestGetWorkflowPermissionsInaccessibleLeavesUnobserved(t *testing.T) { require.Error(t, rest.getWorkflowPermissions()) assert.False(t, rest.WorkflowPermissionsObserved, "insufficient token permissions must leave the observed flag false so the workflow-file heuristic applies") } + +func TestLicenseAtRef(t *testing.T) { + oldAPIBase := APIBase + defer func() { APIBase = oldAPIBase }() + + t.Run("license found at ref", func(t *testing.T) { + var gotRef, gotPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotRef = r.URL.Query().Get("ref") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"path": "LICENSE", "license": {"spdx_id": "MIT"}}`)) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{owner: "test-owner", repo: "test-repo", HttpClient: server.Client()} + license, found, err := rest.LicenseAtRef("v1.0.0") + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, RefLicense{SpdxId: "MIT", Path: "LICENSE"}, license) + assert.Equal(t, "/repos/test-owner/test-repo/license", gotPath) + assert.Equal(t, "v1.0.0", gotRef) + }) + + t.Run("ref is query-escaped", func(t *testing.T) { + var gotRawQuery string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotRawQuery = r.URL.RawQuery + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"path": "LICENSE", "license": {"spdx_id": "MIT"}}`)) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + _, _, err := rest.LicenseAtRef("release/v1 beta") + require.NoError(t, err) + assert.Equal(t, "ref=release%2Fv1+beta", gotRawQuery) + }) + + t.Run("404 means no license at ref, not an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + license, found, err := rest.LicenseAtRef("v1.0.0") + require.NoError(t, err) + assert.False(t, found) + assert.Equal(t, RefLicense{}, license) + }) + + t.Run("403 is reported as ErrRateLimited", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + _, found, err := rest.LicenseAtRef("v1.0.0") + require.Error(t, err) + assert.ErrorIs(t, err, ErrRateLimited) + assert.False(t, found) + }) + + t.Run("non-404/403 failure is an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + _, found, err := rest.LicenseAtRef("v1.0.0") + require.Error(t, err) + assert.False(t, found) + }) + + t.Run("decode failure is an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not-json")) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + _, found, err := rest.LicenseAtRef("v1.0.0") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to decode license data") + assert.False(t, found) + }) +} + +func TestGetLicenseAtRefCachesPerRef(t *testing.T) { + oldAPIBase := APIBase + defer func() { APIBase = oldAPIBase }() + + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"path": "LICENSE", "license": {"spdx_id": "MIT"}}`)) + })) + defer server.Close() + APIBase = server.URL + + payload := Payload{ + RestData: &RestData{HttpClient: server.Client()}, + cache: &payloadCache{}, + } + + for i := 0; i < 2; i++ { + license, found, err := payload.GetLicenseAtRef("v1.0.0") + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "MIT", license.SpdxId) + } + assert.Equal(t, 1, calls, "repeated lookups of the same ref must be cached") + + _, _, err := payload.GetLicenseAtRef("v2.0.0") + require.NoError(t, err) + assert.Equal(t, 2, calls, "a different ref is a different lookup") +} + +func TestRefExists(t *testing.T) { + oldAPIBase := APIBase + defer func() { APIBase = oldAPIBase }() + + t.Run("ref resolves", func(t *testing.T) { + var gotPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{owner: "test-owner", repo: "test-repo", HttpClient: server.Client()} + exists, err := rest.RefExists("v1.0.0") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "/repos/test-owner/test-repo/commits/v1.0.0", gotPath) + }) + + t.Run("ref does not resolve", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + exists, err := rest.RefExists("deleted-tag") + require.NoError(t, err) + assert.False(t, exists) + }) + + t.Run("403 is reported as ErrRateLimited", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer server.Close() + APIBase = server.URL + + rest := &RestData{HttpClient: server.Client()} + _, err := rest.RefExists("v1.0.0") + require.Error(t, err) + assert.ErrorIs(t, err, ErrRateLimited) + }) +} diff --git a/evaluation_plans/osps/build_release/steps.go b/evaluation_plans/osps/build_release/steps.go index 4642afa..eaf2fed 100644 --- a/evaluation_plans/osps/build_release/steps.go +++ b/evaluation_plans/osps/build_release/steps.go @@ -506,16 +506,33 @@ var releaseAssetCompanionSuffixes = []string{ // releaseAssetCompanionNames are exact (lowercased) asset names that accompany // a release without identifying a specific artifact: standard documentation -// files, common split-license names, and the one manifest name the shared -// checksum classifier's markers do not cover. This map matches exactly so a -// real artifact such as license-manager.zip is not exempted; note the shared -// checksum markers it sits alongside are substring-matched, so a name like +// files and the one manifest name the shared checksum classifier's markers do +// not cover. Standalone license files are a separate, exported set below +// (licenseAssetNames) so other controls can reuse the same exact-match +// classification. This map matches exactly so a real artifact such as +// readme-generator.zip is not exempted; note the shared checksum markers it +// sits alongside are substring-matched, so a name like // checksums-generator-1.0.zip is exempted through that path regardless. var releaseAssetCompanionNames = map[string]bool{ "md5sums": true, + "readme": true, "readme.txt": true, "readme.md": true, +} + +// licenseAssetNames are exact (lowercased) asset names recognized as +// standalone license files attached to a release. Exported via +// IsLicenseAssetName so other controls that need to detect a license-shaped +// release asset test the identical set rather than approximating it with a +// prefix match, which would flag real artifacts like +// license-checker_1.0_linux_amd64.tar.gz. +var licenseAssetNames = map[string]bool{ "license": true, "license.txt": true, "license.md": true, "license-mit": true, "license-apache": true, - "readme": true, "readme.txt": true, "readme.md": true, +} + +// IsLicenseAssetName reports whether name (matched case-insensitively) +// is a conventional standalone license file name for a release asset. +func IsLicenseAssetName(name string) bool { + return licenseAssetNames[strings.ToLower(name)] } // isReleaseAssetCompanion reports whether a lowercased asset name is a @@ -529,7 +546,7 @@ func isReleaseAssetCompanion(lowerName string) bool { reusable_steps.HasSBOMExtension(lowerName) { return true } - if releaseAssetCompanionNames[lowerName] { + if releaseAssetCompanionNames[lowerName] || licenseAssetNames[lowerName] { return true } for _, suffix := range releaseAssetCompanionSuffixes { diff --git a/evaluation_plans/osps/legal/steps.go b/evaluation_plans/osps/legal/steps.go index 5c607ae..460497f 100644 --- a/evaluation_plans/osps/legal/steps.go +++ b/evaluation_plans/osps/legal/steps.go @@ -2,11 +2,14 @@ package legal import ( "encoding/json" + "errors" "fmt" "strings" + "time" "github.com/gemaraproj/go-gemara" "github.com/ossf/pvtr-github-repo-scanner/data" + "github.com/ossf/pvtr-github-repo-scanner/evaluation_plans/osps/build_release" ) type LicenseList struct { @@ -117,15 +120,163 @@ func FoundLicense(payload data.Payload) (result gemara.Result, message string, c return gemara.Failed, "License was not found in a well known location via the GitHub API", gemara.Medium } +// latestPublishedRelease returns the most recently published non-draft, +// non-prerelease release. GitHub's /releases listing is ordered by creation +// time, not publish time, and can include prereleases, so this selects +// explicitly by parsed PublishedAt rather than trusting list order or taking +// the first non-draft entry. +func latestPublishedRelease(releases []data.ReleaseData) (data.ReleaseData, bool) { + var latest data.ReleaseData + var latestTime time.Time + found := false + for _, release := range releases { + if release.Draft || release.Prerelease { + continue + } + published, err := time.Parse(time.RFC3339, release.PublishedAt) + if err != nil { + // Unparsable or missing timestamp; it can't be compared against + // candidates, so it can't be selected as "latest". + continue + } + if !found || published.After(latestTime) { + latest = release + latestTime = published + found = true + } + } + return latest, found +} + +// licenseAssets returns the names of release assets that look like standalone +// license files attached at the top level of the release. It reuses +// build_release's exact-match classifier so the same asset is never +// classified as a license file by one control and a real artifact by another. +func licenseAssets(release data.ReleaseData) []string { + var names []string + for _, asset := range release.Assets { + if build_release.IsLicenseAssetName(asset.Name) { + names = append(names, asset.Name) + } + } + return names +} + +// releaseLabel names a release for messages, preferring the tag over the +// free-form release name. +func releaseLabel(release data.ReleaseData) string { + if release.TagName != "" { + return release.TagName + } + return release.Name +} + +// ReleasesLicensed assesses whether released software assets include their +// license (OSPS-LE-03.02, and the license-presence half of OSPS-LE-02.02). +// +// The assessment is scoped to the latest published release, which reflects the +// project's current licensing posture. Two release-time hazards drive the +// design (see ossf/pvtr-github-repo-scanner#70): +// +// 1. The license can be modified or removed at the release tag while the +// default branch still shows an approved one. GitHub's auto-generated +// source archives contain the tree at the tag, so the authoritative +// evidence is the license GitHub detects at that tag, not at HEAD. This is +// checked first and, when conclusive, decides the result outright. +// 2. A license file attached directly to the release can supersede whatever +// ships inside the source archives, but its content is not observable +// here. It is treated as corroborating evidence, not a blocker: it never +// downgrades a tag-level Pass or Fail, and only tips an otherwise +// inconclusive result toward NeedsReview rather than Failed. +// +// Default-branch evidence is used only as a lower-confidence fallback when +// there is no tag name to check directly. A tag-level lookup that fails +// outright (rather than resolving to found/not-found) is not treated as +// "unavailable" in that sense — see the default case below — because #70's +// whole premise is that HEAD can look licensed while the tag is not. func ReleasesLicensed(payload data.Payload) (result gemara.Result, message string, confidence gemara.ConfidenceLevel) { - if len(payload.Releases) == 0 { - return gemara.NotApplicable, "No releases found", confidence + if payload.RestData == nil { + return gemara.NeedsReview, "Release data is unavailable; review the released assets for license coverage", gemara.Low } - if payload.Repository.LicenseInfo.Url != "" { - return gemara.Passed, "GitHub releases include the license(s) in the released source code.", gemara.High + if payload.ReleasesError != nil { + return gemara.NeedsReview, fmt.Sprintf("Release data could not be retrieved: %v. Review the released assets for license coverage", payload.ReleasesError), gemara.Low + } + latest, ok := latestPublishedRelease(payload.Releases) + if !ok { + return gemara.NotApplicable, "No releases found", gemara.High + } + assets := licenseAssets(latest) + + if latest.TagName != "" { + license, found, err := payload.GetLicenseAtRef(latest.TagName) + switch { + case err == nil && found && license.SpdxId != "" && license.SpdxId != noAssertion: + return gemara.Passed, fmt.Sprintf("GitHub identifies license %s at %q in the released source code at tag %q; the auto-generated release archives include it", license.SpdxId, license.Path, latest.TagName), gemara.High + case err == nil && found: + return gemara.Passed, fmt.Sprintf("License file %q is present in the released source code at tag %q, but GitHub could not identify the license type", license.Path, latest.TagName), gemara.Medium + case err == nil: + // GitHub found no license file at the tag. That reading alone is + // ambiguous: a 404 from this endpoint means the same thing whether + // the tag genuinely ships no recognized license file or the tag no + // longer resolves to a commit at all (deleted or re-pointed after + // the release was published), so the ref's resolvability is + // checked before treating this as a real absence. + exists, refErr := payload.RefExists(latest.TagName) + switch { + case refErr != nil: + return gemara.NeedsReview, fmt.Sprintf("Could not confirm tag %q still resolves to a commit; review the released source code for license coverage", latest.TagName), gemara.Low + case !exists: + return gemara.NeedsReview, fmt.Sprintf("Tag %q no longer resolves to a commit, so its released source code could not be checked directly; manual review is required", latest.TagName), gemara.Low + } + if len(assets) > 0 { + return gemara.NeedsReview, fmt.Sprintf("No license was found in the released source code at tag %q, but release %q attaches standalone license file(s) as assets (%s); manual review is required to confirm they cover the release", latest.TagName, releaseLabel(latest), strings.Join(assets, ", ")), gemara.Medium + } + if payload.GraphqlRepoData != nil && payload.Repository.LicenseInfo.Url != "" { + // The tamper/removal case #70 exists to catch: HEAD is + // licensed, but the tree actually shipped is not. + return gemara.Failed, fmt.Sprintf("A license exists on the default branch, but none was found in the released source code at tag %q", latest.TagName), gemara.Medium + } + return gemara.Failed, fmt.Sprintf("No license was found in the released source code at tag %q", latest.TagName), gemara.Medium + default: + // Any failure other than "checked and found nothing" — a network + // error, a rate-limited/permission-denied 403, a 5xx from GitHub — + // means the released tree could not be checked directly at all. + // #70 exists specifically to catch a tag whose license was + // tampered with or removed while HEAD still looks licensed, so + // falling back to HEAD evidence here would silently defeat that + // check exactly when it matters most (e.g. mid rate-limited bulk + // scan). This gets the same disposition as a failed /releases + // fetch: NeedsReview, never a fallback Pass. + if payload.Config != nil && payload.Config.Logger != nil { + payload.Config.Logger.Warn(fmt.Sprintf("could not check the released source code at tag %q for a license: %s", latest.TagName, err.Error())) + } + detail := "the request could not be completed" + if errors.Is(err, data.ErrRateLimited) { + // 403 covers both rate-limit exhaustion and a token that + // simply lacks permission; the disposition is the same + // either way, so the wording doesn't overclaim which one it was. + detail = "the request was denied (rate limited or insufficient permission)" + } + if len(assets) > 0 { + return gemara.NeedsReview, fmt.Sprintf("Could not check the released source code at tag %q for a license (%s); release %q attaches standalone license file(s) as assets (%s), but that could not be confirmed either, so manual review is required", latest.TagName, detail, releaseLabel(latest), strings.Join(assets, ", ")), gemara.Low + } + return gemara.NeedsReview, fmt.Sprintf("Could not check the released source code at tag %q for a license: %s. Review the released assets for license coverage", latest.TagName, detail), gemara.Low + } + } + + if len(assets) > 0 { + return gemara.NeedsReview, fmt.Sprintf("Release %q attaches standalone license file(s) as release assets (%s); the released source code could not be checked directly to confirm they match, so manual review is required", releaseLabel(latest), strings.Join(assets, ", ")), gemara.Medium + } + + // Fallback: the release has no tag name, so there is no ref to check + // directly, and the default branch stands in as a weaker proxy for what + // the release archives contain. A tag-level lookup failure no longer + // reaches this point — see the default case above. + if payload.GraphqlRepoData != nil && payload.Repository.LicenseInfo.Url != "" { + return gemara.Passed, "A license was found on the default branch; the released source code could not be checked directly, so this is default-branch evidence only", gemara.Medium } if file := findRootLicenseFile(payload); file != "" { - return gemara.Passed, fmt.Sprintf("License file %q found in the repository root; GitHub could not identify the license type", file), gemara.Medium + return gemara.Passed, fmt.Sprintf("License file %q found in the repository root; the released source code could not be checked directly, so this is default-branch evidence only", file), gemara.Low } return gemara.Failed, "License was not found in a well known location via the GitHub API", gemara.Medium } diff --git a/evaluation_plans/osps/legal/steps_test.go b/evaluation_plans/osps/legal/steps_test.go index 11dadb6..9c032d4 100644 --- a/evaluation_plans/osps/legal/steps_test.go +++ b/evaluation_plans/osps/legal/steps_test.go @@ -2,6 +2,9 @@ package legal import ( "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" "github.com/gemaraproj/go-gemara" @@ -104,77 +107,292 @@ func TestFoundLicense(t *testing.T) { } } +// withLicenseEndpoint points data.APIBase at a stub license endpoint for the +// duration of the test. statusCode 404 simulates "no license at the ref"; +// 200 responds with the given SPDX id and path; anything else is a hard error. +// Requests to /commits/ (the ref-resolvability check) always answer 200, +// i.e. "the ref resolves", unless overridden by refExistsStatus. +func withLicenseEndpoint(t *testing.T, statusCode int, spdxId, path string) { + withLicenseAndRefEndpoint(t, statusCode, spdxId, path, http.StatusOK) +} + +func withLicenseAndRefEndpoint(t *testing.T, licenseStatus int, spdxId, path string, refExistsStatus int) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/commits/") { + w.WriteHeader(refExistsStatus) + return + } + if licenseStatus != http.StatusOK { + w.WriteHeader(licenseStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"path": %q, "license": {"spdx_id": %q}}`, path, spdxId) + })) + t.Cleanup(server.Close) + oldAPIBase := data.APIBase + data.APIBase = server.URL + t.Cleanup(func() { data.APIBase = oldAPIBase }) +} + +// restDataWithReleases builds a RestData carrying the given releases and a +// working HTTP client for tag-level license lookups. +func restDataWithReleases(releases ...data.ReleaseData) *data.RestData { + return &data.RestData{ + Releases: releases, + HttpClient: http.DefaultClient, + } +} + func TestReleasesLicensed(t *testing.T) { - tests := []struct { - name string - payload data.Payload - expectedResult gemara.Result - expectedMessage string - }{ - { - name: "No releases found", - payload: data.Payload{ - RestData: &data.RestData{ - Releases: []data.ReleaseData{}, - }, - }, - expectedResult: gemara.NotApplicable, - expectedMessage: "No releases found", - }, - { - name: "No licenses found", - payload: data.Payload{ - RestData: &data.RestData{ - Releases: []data.ReleaseData{ - { - Name: "v1.0.0", - }, - }, - }, - GraphqlRepoData: &data.GraphqlRepoData{}, - }, - expectedResult: gemara.Failed, - expectedMessage: "License was not found in a well known location via the GitHub API", - }, - { - name: "Has releases and license", - payload: data.Payload{ - RestData: &data.RestData{ - Releases: []data.ReleaseData{ - { - Name: "v1.0.0", - }, - }, - }, - GraphqlRepoData: stubGraphqlRepo("https://api.github.com/licenses/mit"), - }, - expectedResult: gemara.Passed, - expectedMessage: "GitHub releases include the license(s) in the released source code.", - }, - { - name: "Release with unclassified root license file", - payload: data.Payload{ - RestData: &data.RestData{ - Releases: []data.ReleaseData{ - { - Name: "v1.0.0", - }, - }, + t.Run("no RestData", func(t *testing.T) { + result, message, confidence := ReleasesLicensed(data.Payload{}) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, "Release data is unavailable; review the released assets for license coverage", message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("release fetch failed", func(t *testing.T) { + payload := data.Payload{ + RestData: &data.RestData{ReleasesError: fmt.Errorf("boom")}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, "Release data could not be retrieved: boom. Review the released assets for license coverage", message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("no releases found", func(t *testing.T) { + payload := data.Payload{RestData: &data.RestData{}} + result, message, _ := ReleasesLicensed(payload) + assert.Equal(t, gemara.NotApplicable, result) + assert.Equal(t, "No releases found", message) + }) + + t.Run("draft releases do not count as published", func(t *testing.T) { + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0", Draft: true, PublishedAt: "2026-01-01T00:00:00Z"}), + } + result, message, _ := ReleasesLicensed(payload) + assert.Equal(t, gemara.NotApplicable, result) + assert.Equal(t, "No releases found", message) + }) + + t.Run("prereleases do not count as published", func(t *testing.T) { + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0-rc1", Prerelease: true, PublishedAt: "2026-01-01T00:00:00Z"}), + } + result, message, _ := ReleasesLicensed(payload) + assert.Equal(t, gemara.NotApplicable, result) + assert.Equal(t, "No releases found", message) + }) + + t.Run("only the most recently published release is considered, regardless of list order", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusOK, "MIT", "LICENSE") + payload := data.Payload{ + RestData: restDataWithReleases( + data.ReleaseData{TagName: "v2.0.0-rc1", Prerelease: true, PublishedAt: "2026-03-01T00:00:00Z"}, + data.ReleaseData{TagName: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}, + data.ReleaseData{TagName: "v1.1.0", PublishedAt: "2026-02-01T00:00:00Z"}, + ), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Passed, result) + assert.Equal(t, `GitHub identifies license MIT at "LICENSE" in the released source code at tag "v1.1.0"; the auto-generated release archives include it`, message) + assert.Equal(t, gemara.High, confidence) + }) + + t.Run("license identified at the release tag passes regardless of an attached license asset", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusOK, "Apache-2.0", "LICENSE") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{ + TagName: "v1.0.0", + PublishedAt: "2026-01-01T00:00:00Z", + Assets: []data.ReleaseAsset{{Name: "LICENSE.txt"}}, + }), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Passed, result) + assert.Equal(t, `GitHub identifies license Apache-2.0 at "LICENSE" in the released source code at tag "v1.0.0"; the auto-generated release archives include it`, message) + assert.Equal(t, gemara.High, confidence) + }) + + t.Run("unidentified license at the release tag", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusOK, "NOASSERTION", "COPYING") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Passed, result) + assert.Equal(t, `License file "COPYING" is present in the released source code at tag "v1.0.0", but GitHub could not identify the license type`, message) + assert.Equal(t, gemara.Medium, confidence) + }) + + t.Run("no license at tag, but a license asset is attached needs review", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusNotFound, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{ + TagName: "v1.0.0", + PublishedAt: "2026-01-01T00:00:00Z", + Assets: []data.ReleaseAsset{ + {Name: "scanner-linux-amd64.tar.gz"}, + {Name: "LICENSE.txt"}, + {Name: "LICENSE-MIT"}, }, - GraphqlRepoData: stubGraphqlRepoWithTree("", treeEntry{name: "LICENSE"}), - }, - expectedResult: gemara.Passed, - expectedMessage: `License file "LICENSE" found in the repository root; GitHub could not identify the license type`, - }, - } + }), + GraphqlRepoData: stubGraphqlRepo("https://api.github.com/licenses/mit"), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, `No license was found in the released source code at tag "v1.0.0", but release "v1.0.0" attaches standalone license file(s) as assets (LICENSE.txt, LICENSE-MIT); manual review is required to confirm they cover the release`, message) + assert.Equal(t, gemara.Medium, confidence) + }) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result, message, _ := ReleasesLicensed(test.payload) - assert.Equal(t, test.expectedResult, result) - assert.Equal(t, test.expectedMessage, message) - }) - } + t.Run("release asset name that merely contains 'license-' is not misclassified as a license file", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusNotFound, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{ + TagName: "v1.0.0", + PublishedAt: "2026-01-01T00:00:00Z", + Assets: []data.ReleaseAsset{{Name: "license-checker_1.0_linux_amd64.tar.gz"}}, + }), + GraphqlRepoData: &data.GraphqlRepoData{}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Failed, result) + assert.Equal(t, `No license was found in the released source code at tag "v1.0.0"`, message) + assert.Equal(t, gemara.Medium, confidence) + }) + + t.Run("license on default branch but missing at the release tag", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusNotFound, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: stubGraphqlRepo("https://api.github.com/licenses/mit"), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Failed, result) + assert.Equal(t, `A license exists on the default branch, but none was found in the released source code at tag "v1.0.0"`, message) + assert.Equal(t, gemara.Medium, confidence) + }) + + t.Run("no license anywhere at the release tag", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusNotFound, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: &data.GraphqlRepoData{}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Failed, result) + assert.Equal(t, `No license was found in the released source code at tag "v1.0.0"`, message) + assert.Equal(t, gemara.Medium, confidence) + }) + + t.Run("tag no longer resolves to a commit needs review, not Failed", func(t *testing.T) { + withLicenseAndRefEndpoint(t, http.StatusNotFound, "", "", http.StatusNotFound) + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "deleted-tag", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: &data.GraphqlRepoData{}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, `Tag "deleted-tag" no longer resolves to a commit, so its released source code could not be checked directly; manual review is required`, message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("rate limited tag lookup needs review, does not fall back to default-branch evidence", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusForbidden, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: stubGraphqlRepo("https://api.github.com/licenses/mit"), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, `Could not check the released source code at tag "v1.0.0" for a license: the request was denied (rate limited or insufficient permission). Review the released assets for license coverage`, message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("generic tag lookup failure needs review, does not fall back to default-branch evidence", func(t *testing.T) { + // Regression guard: a tag-level lookup failure must never fall + // through to a Passed verdict on (potentially stale) default-branch + // evidence, whether the failure is a 403 or anything else — #70 + // exists to catch exactly the case where HEAD still looks licensed + // but the tag does not. + withLicenseEndpoint(t, http.StatusInternalServerError, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{TagName: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: stubGraphqlRepo("https://api.github.com/licenses/mit"), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, `Could not check the released source code at tag "v1.0.0" for a license: the request could not be completed. Review the released assets for license coverage`, message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("generic tag lookup failure with an attached license asset still needs review, not a corroborated pass", func(t *testing.T) { + withLicenseEndpoint(t, http.StatusInternalServerError, "", "") + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{ + TagName: "v1.0.0", + PublishedAt: "2026-01-01T00:00:00Z", + Assets: []data.ReleaseAsset{{Name: "LICENSE.txt"}}, + }), + GraphqlRepoData: &data.GraphqlRepoData{}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, `Could not check the released source code at tag "v1.0.0" for a license (the request could not be completed); release "v1.0.0" attaches standalone license file(s) as assets (LICENSE.txt), but that could not be confirmed either, so manual review is required`, message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("release without a tag name falls back to default-branch license", func(t *testing.T) { + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{Name: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: stubGraphqlRepo("https://api.github.com/licenses/mit"), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Passed, result) + assert.Equal(t, "A license was found on the default branch; the released source code could not be checked directly, so this is default-branch evidence only", message) + assert.Equal(t, gemara.Medium, confidence) + }) + + t.Run("release without a tag name falls back to unclassified root license file", func(t *testing.T) { + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{Name: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: stubGraphqlRepoWithTree("", treeEntry{name: "LICENSE"}), + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Passed, result) + assert.Equal(t, `License file "LICENSE" found in the repository root; the released source code could not be checked directly, so this is default-branch evidence only`, message) + assert.Equal(t, gemara.Low, confidence) + }) + + t.Run("release without a tag name and no license evidence anywhere fails", func(t *testing.T) { + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{Name: "v1.0.0", PublishedAt: "2026-01-01T00:00:00Z"}), + GraphqlRepoData: &data.GraphqlRepoData{}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.Failed, result) + assert.Equal(t, "License was not found in a well known location via the GitHub API", message) + assert.Equal(t, gemara.Medium, confidence) + }) + + t.Run("release without a tag name but with an attached license asset needs review", func(t *testing.T) { + payload := data.Payload{ + RestData: restDataWithReleases(data.ReleaseData{ + Name: "v1.0.0", + PublishedAt: "2026-01-01T00:00:00Z", + Assets: []data.ReleaseAsset{{Name: "LICENSE.txt"}}, + }), + GraphqlRepoData: &data.GraphqlRepoData{}, + } + result, message, confidence := ReleasesLicensed(payload) + assert.Equal(t, gemara.NeedsReview, result) + assert.Equal(t, `Release "v1.0.0" attaches standalone license file(s) as release assets (LICENSE.txt); the released source code could not be checked directly to confirm they match, so manual review is required`, message) + assert.Equal(t, gemara.Medium, confidence) + }) } func TestGetLicenseList(t *testing.T) {