diff --git a/.golangci.yml b/.golangci.yml index a2df940..4110063 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -83,3 +83,4 @@ linters: - cyclop - godot - funlen + - lll diff --git a/CHANGELOG.md b/CHANGELOG.md index f845500..77fc344 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.6.4] - 2026-03-23 +### Added +- Add support for single worker scan timeout HTTP response handling (504) + ## [1.6.3] - 2026-03-10 ### Added - Add dynamic support for loading env vars (from file) during startup. @@ -195,3 +199,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.6.1]: https://github.com/scanoss/api.go/compare/v1.6.0...v1.6.1 [1.6.2]: https://github.com/scanoss/api.go/compare/v1.6.1...v1.6.2 [1.6.3]: https://github.com/scanoss/api.go/compare/v1.6.2...v1.6.3 +[1.6.4]: https://github.com/scanoss/api.go/compare/v1.6.3...v1.6.4 diff --git a/pkg/service/kb_details.go b/pkg/service/kb_details.go index 423a208..1d39382 100644 --- a/pkg/service/kb_details.go +++ b/pkg/service/kb_details.go @@ -111,7 +111,7 @@ func (s APIService) loadKBDetails() { } // Load a random (hopefully non-existent) file match to extract the KB version details emptyConfig := DefaultScanningServiceConfig(s.config) - result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", emptyConfig, zs) + result, _, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", emptyConfig, zs) if err != nil { zs.Warnf("Failed to detect KB version from eninge: %v", err) return diff --git a/pkg/service/scanning_service.go b/pkg/service/scanning_service.go index 653a1ac..985a9d8 100644 --- a/pkg/service/scanning_service.go +++ b/pkg/service/scanning_service.go @@ -231,10 +231,14 @@ func (s APIService) writeSbomFile(sbom string, zs *zap.SugaredLogger) (*os.File, // singleScan runs a scan of the WFP in a single thread. func (s APIService) singleScan(wfp, sbomFile string, config ScanningServiceConfig, zs *zap.SugaredLogger, w http.ResponseWriter) { zs.Debugf("Single threaded scan...") - result, err := s.scanWfp(wfp, sbomFile, config, zs) + result, timedOut, err := s.scanWfp(wfp, sbomFile, config, zs) if err != nil { + if timedOut { + http.Error(w, "ERROR engine scan timed out", http.StatusGatewayTimeout) + } else { + http.Error(w, "ERROR engine scan failed", http.StatusInternalServerError) + } zs.Errorf("Engine scan failed: %v", err) - http.Error(w, "ERROR engine scan failed", http.StatusInternalServerError) } else { zs.Debug("Scan completed") response := strings.TrimSpace(result) @@ -355,7 +359,7 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str zs.Warnf("Nothing in the job request to scan. Ignoring") results <- "" } else { - result, err := s.scanWfp(job, sbomFile, config, zs) + result, _, err := s.scanWfp(job, sbomFile, config, zs) if s.config.App.Trace { zs.Debugf("scan result (%v): %v, %v", id, result, err) } @@ -380,15 +384,15 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str } // scanWfp run the scanoss engine scan of the supplied WFP. -func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, zs *zap.SugaredLogger) (string, error) { +func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, zs *zap.SugaredLogger) (string, bool, error) { if len(wfp) == 0 { zs.Warnf("Nothing in the job request to scan. Ignoring") - return "", fmt.Errorf("no wfp supplied to scan. ignoring") + return "", false, fmt.Errorf("no wfp supplied to scan. ignoring") } tempFile, err := os.CreateTemp(s.config.Scanning.WfpLoc, "finger*.wfp") if err != nil { zs.Errorf("Failed to create temporary file: %v", err) - return "", fmt.Errorf("failed to create temporary WFP file") + return "", false, fmt.Errorf("failed to create temporary WFP file") } if s.config.Scanning.TmpFileDelete { defer removeFile(tempFile, zs) @@ -398,7 +402,7 @@ func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, if err != nil { closeFile(tempFile, zs) zs.Errorf("Failed to write WFP to temporary file: %v", err) - return "", fmt.Errorf("failed to write to temporary WFP file") + return "", false, fmt.Errorf("failed to write to temporary WFP file") } closeFile(tempFile, zs) // Build command arguments @@ -448,11 +452,13 @@ func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, timeoutErr := fmt.Errorf("scan command timed out after %v seconds", s.config.Scanning.ScanTimeout) ctx, cancel := context.WithTimeoutCause(context.Background(), time.Duration(s.config.Scanning.ScanTimeout)*time.Second, timeoutErr) // put a timeout on the scan execution defer cancel() + timeoutEncountered := false //nolint:gosec output, err := exec.CommandContext(ctx, s.config.Scanning.ScanBinary, args...).Output() if err != nil { if cause := context.Cause(ctx); cause != nil { zs.Errorf("Scan command (%v) timed out: %v", s.config.Scanning.ScanBinary, cause) + timeoutEncountered = true } else { zs.Errorf("Scan command (%v %v) failed: %v", s.config.Scanning.ScanBinary, args, err) } @@ -460,9 +466,9 @@ func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, if s.config.Scanning.KeepFailedWfps { s.copyWfpTempFile(tempFile.Name(), zs) } - return "", fmt.Errorf("failed to scan WFP: %v", err) + return "", timeoutEncountered, fmt.Errorf("failed to scan WFP: %v", err) } - return string(output), err + return string(output), timeoutEncountered, err } // TestEngine tests if the SCANOSS engine is accessible and running. diff --git a/pkg/service/scanning_service_test.go b/pkg/service/scanning_service_test.go index 83e813c..21c845f 100644 --- a/pkg/service/scanning_service_test.go +++ b/pkg/service/scanning_service_test.go @@ -112,17 +112,21 @@ func TestScanDirectSingle(t *testing.T) { myConfig := setupConfig(t) myConfig.App.Trace = true myConfig.Scanning.ScanDebug = true + myConfig.Scanning.MatchConfigAllowed = false + myConfig.Scanning.RankingAllowed = false apiService := NewAPIService(myConfig) tests := []struct { - name string - fieldName string - file string - binary string - telemetry bool - scanType string - assets string - want int + name string + fieldName string + file string + binary string + telemetry bool + scanType string + assets string + scanSettingsB64 string + settingsAllowed bool + want int }{ { name: "Scanning - wrong name", @@ -186,6 +190,77 @@ func TestScanDirectSingle(t *testing.T) { assets: "pkg:github/org/repo", want: http.StatusOK, }, + { + name: "Scanning - Settings - invalid base64", + binary: "../../test-support/scanoss.sh", + fieldName: "filename", + file: "./tests/fingers.wfp", + settingsAllowed: true, + scanSettingsB64: "invalid-base64!!!", + want: http.StatusBadRequest, + }, + { + name: "Scanning - Settings - invalid json", + binary: "../../test-support/scanoss.sh", + fieldName: "filename", + file: "./tests/fingers.wfp", + // Base64 decoded JSON: + // { + // "field": "something, + // "array": [ + // } + scanSettingsB64: "ewoiZmllbGQiOiAic29tZXRoaW5nLAogImFycmF5IjogWwp9", + settingsAllowed: true, + want: http.StatusBadRequest, + }, + { + name: "Scanning - Settings - not allowed", + binary: "../../test-support/scanoss.sh", + fieldName: "filename", + file: "./tests/fingers.wfp", + // Base64 decoded JSON: + // { + // "min_snippet_hits": 5, + // "min_snippet_lines": 10 + // } + scanSettingsB64: "eyJtaW5fc25pcHBldF9oaXRzIjo1LCJtaW5fc25pcHBldF9saW5lcyI6MTB9", + settingsAllowed: false, + want: http.StatusBadRequest, + }, + { + name: "Scanning - Settings - success 1", + binary: "../../test-support/scanoss.sh", + fieldName: "filename", + file: "./tests/fingers.wfp", + // Base64 decoded JSON: + // { + // "ranking_enabled": true, + // "ranking_threshold": 85, + // "min_snippet_hits": 3, + // "min_snippet_lines": 8, + // "honour_file_exts": false + // } + scanSettingsB64: "eyJyYW5raW5nX2VuYWJsZWQiOnRydWUsInJhbmtpbmdfdGhyZXNob2xkIjo4NSwibWluX3NuaXBwZXRfaGl0cyI6MywibWluX3NuaXBwZXRfbGluZXMiOjgsImhvbm91cl9maWxlX2V4dHMiOmZhbHNlfQ==", + settingsAllowed: true, + want: http.StatusOK, + }, + { + name: "Scanning - Settings - success 2", + binary: "../../test-support/scanoss.sh", + fieldName: "filename", + file: "./tests/fingers.wfp", + // Base64 decoded JSON: + // { + // "ranking_enabled": true, + // "ranking_threshold": -1, + // "min_snippet_hits": 3, + // "min_snippet_lines": 8, + // "honour_file_exts": true + // } + scanSettingsB64: "ewogICJyYW5raW5nX2VuYWJsZWQiOiB0cnVlLAogICJyYW5raW5nX3RocmVzaG9sZCI6IC0xLAogICJtaW5fc25pcHBldF9oaXRzIjogMywKICAibWluX3NuaXBwZXRfbGluZXMiOiA4LAogICJob25vdXJfZmlsZV9leHRzIjogdHJ1ZQp9", + settingsAllowed: true, + want: http.StatusOK, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -196,6 +271,8 @@ func TestScanDirectSingle(t *testing.T) { myConfig.App.Trace = true } } + myConfig.Scanning.MatchConfigAllowed = test.settingsAllowed + myConfig.Scanning.RankingEnabled = test.settingsAllowed myConfig.Scanning.ScanBinary = test.binary myConfig.Telemetry.Enabled = test.telemetry filePath := test.file @@ -225,9 +302,11 @@ func TestScanDirectSingle(t *testing.T) { } } _ = mw.Close() // close the writer before making the request - req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody) w := httptest.NewRecorder() + if len(test.scanSettingsB64) > 0 { + req.Header.Set("Scanoss-Settings", test.scanSettingsB64) + } req.Header.Add("Content-Type", mw.FormDataContentType()) apiService.ScanDirect(w, req) resp := w.Result() @@ -449,3 +528,76 @@ func TestScanDirectSingleHPSM(t *testing.T) { }) } } + +func TestScanDirectSingleSlow(t *testing.T) { + err := zlog.NewSugaredDevLogger() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) + } + defer zlog.SyncZap() + myConfig := setupConfig(t) + myConfig.App.Trace = true + myConfig.Scanning.ScanDebug = true + myConfig.Scanning.ScanTimeout = 5 + apiService := NewAPIService(myConfig) + + tests := []struct { + name string + fieldName string + file string + binary string + scanType string + assets string + want int + }{ + { + name: "Scanning - success 1", + binary: "../../test-support/scanoss.sh", + fieldName: "file", + file: "./tests/fingers.wfp", + want: http.StatusOK, + }, + { + name: "Scanning - Slow fail", + binary: "../../test-support/scanoss-slow.sh", + fieldName: "filename", + file: "./tests/fingers-hpsm.wfp", + want: http.StatusGatewayTimeout, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + myConfig.Scanning.ScanBinary = test.binary + filePath := test.file + fieldName := test.fieldName + postBody := new(bytes.Buffer) + mw := multipart.NewWriter(postBody) + file, err := os.Open(filePath) + if err != nil { + t.Fatal(err) + } + writer, err := mw.CreateFormFile(fieldName, filePath) + if err != nil { + t.Fatal(err) + } + if _, err = io.Copy(writer, file); err != nil { + t.Fatal(err) + } + _ = mw.Close() // close the writer before making the request + + req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody) + w := httptest.NewRecorder() + req.Header.Add("Content-Type", mw.FormDataContentType()) + apiService.ScanDirect(w, req) + resp := w.Result() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("an error was not expected when reading from request: %v", err) + } + assert.Equal(t, test.want, resp.StatusCode) + fmt.Println("Status: ", resp.StatusCode) + fmt.Println("Type: ", resp.Header.Get("Content-Type")) + fmt.Println("Body: ", string(body)) + }) + } +} diff --git a/test-support/scanoss-slow.sh b/test-support/scanoss-slow.sh new file mode 100755 index 0000000..36bf035 --- /dev/null +++ b/test-support/scanoss-slow.sh @@ -0,0 +1,33 @@ +#!/bin/bash +### +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2018-2023 SCANOSS.COM +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +### +d=$(dirname "$0") +# Simulate getting file contents +if [ "$1" == "-h" ] || [ "$2" == "-h" ] || [ "$1" == "-help" ] || [ "$2" == "-help" ] ; then + echo "SCANOSS slow engine simulator help" + echo " command options..." + exit 0 +fi +export DELAY=10 +echo "Info: Running slow simulation delay: $DELAY" >&2 +"$d"/scanoss.sh "$@" +EXIT_CODE=$? +# Only sleep if the command finished successfully +if [ $EXIT_CODE -eq 0 ] ; then + sleep $DELAY +fi +exit $EXIT_CODE