Add timeout 504 handling for single file, single worker scan requests

.golangci.yml

@@ -83,3 +83,4 @@ linters:
           - cyclop
           - godot
           - funlen
+          - lll

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.6.4] - 2026-03-23
+### Added
+- Add support for single worker scan timeout HTTP response handling (504)
+
 ## [1.6.3] - 2026-03-10
 ### Added
 - Add dynamic support for loading env vars (from file) during startup.
@@ -195,3 +199,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.6.1]: https://github.com/scanoss/api.go/compare/v1.6.0...v1.6.1
 [1.6.2]: https://github.com/scanoss/api.go/compare/v1.6.1...v1.6.2
 [1.6.3]: https://github.com/scanoss/api.go/compare/v1.6.2...v1.6.3
+[1.6.4]: https://github.com/scanoss/api.go/compare/v1.6.3...v1.6.4

pkg/service/kb_details.go

@@ -111,7 +111,7 @@ func (s APIService) loadKBDetails() {
 	}
 	// Load a random (hopefully non-existent) file match to extract the KB version details
 	emptyConfig := DefaultScanningServiceConfig(s.config)
-	result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", emptyConfig, zs)
+	result, _, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", emptyConfig, zs)
 	if err != nil {
 		zs.Warnf("Failed to detect KB version from eninge: %v", err)
 		return

pkg/service/scanning_service.go

@@ -231,10 +231,14 @@ func (s APIService) writeSbomFile(sbom string, zs *zap.SugaredLogger) (*os.File,
 // singleScan runs a scan of the WFP in a single thread.
 func (s APIService) singleScan(wfp, sbomFile string, config ScanningServiceConfig, zs *zap.SugaredLogger, w http.ResponseWriter) {
 	zs.Debugf("Single threaded scan...")
-	result, err := s.scanWfp(wfp, sbomFile, config, zs)
+	result, timedOut, err := s.scanWfp(wfp, sbomFile, config, zs)
 	if err != nil {
+		if timedOut {
+			http.Error(w, "ERROR engine scan timed out", http.StatusGatewayTimeout)
+		} else {
+			http.Error(w, "ERROR engine scan failed", http.StatusInternalServerError)
+		}
 		zs.Errorf("Engine scan failed: %v", err)
-		http.Error(w, "ERROR engine scan failed", http.StatusInternalServerError)
 	} else {
 		zs.Debug("Scan completed")
 		response := strings.TrimSpace(result)
@@ -355,7 +359,7 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str
 			zs.Warnf("Nothing in the job request to scan. Ignoring")
 			results <- ""
 		} else {
-			result, err := s.scanWfp(job, sbomFile, config, zs)
+			result, _, err := s.scanWfp(job, sbomFile, config, zs)
 			if s.config.App.Trace {
 				zs.Debugf("scan result (%v): %v, %v", id, result, err)
 			}
@@ -380,15 +384,15 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str
 }
 
 // scanWfp run the scanoss engine scan of the supplied WFP.
-func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, zs *zap.SugaredLogger) (string, error) {
+func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig, zs *zap.SugaredLogger) (string, bool, error) {
 	if len(wfp) == 0 {
 		zs.Warnf("Nothing in the job request to scan. Ignoring")
-		return "", fmt.Errorf("no wfp supplied to scan. ignoring")
+		return "", false, fmt.Errorf("no wfp supplied to scan. ignoring")
 	}
 	tempFile, err := os.CreateTemp(s.config.Scanning.WfpLoc, "finger*.wfp")
 	if err != nil {
 		zs.Errorf("Failed to create temporary file: %v", err)
-		return "", fmt.Errorf("failed to create temporary WFP file")
+		return "", false, fmt.Errorf("failed to create temporary WFP file")
 	}
 	if s.config.Scanning.TmpFileDelete {
 		defer removeFile(tempFile, zs)
@@ -398,7 +402,7 @@ func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig,
 	if err != nil {
 		closeFile(tempFile, zs)
 		zs.Errorf("Failed to write WFP to temporary file: %v", err)
-		return "", fmt.Errorf("failed to write to temporary WFP file")
+		return "", false, fmt.Errorf("failed to write to temporary WFP file")
 	}
 	closeFile(tempFile, zs)
 	// Build command arguments
@@ -448,21 +452,23 @@ func (s APIService) scanWfp(wfp, sbomFile string, config ScanningServiceConfig,
 	timeoutErr := fmt.Errorf("scan command timed out after %v seconds", s.config.Scanning.ScanTimeout)
 	ctx, cancel := context.WithTimeoutCause(context.Background(), time.Duration(s.config.Scanning.ScanTimeout)*time.Second, timeoutErr) // put a timeout on the scan execution
 	defer cancel()
+	timeoutEncountered := false
 	//nolint:gosec
 	output, err := exec.CommandContext(ctx, s.config.Scanning.ScanBinary, args...).Output()
 	if err != nil {
 		if cause := context.Cause(ctx); cause != nil {
 			zs.Errorf("Scan command (%v) timed out: %v", s.config.Scanning.ScanBinary, cause)
+			timeoutEncountered = true
 		} else {
 			zs.Errorf("Scan command (%v %v) failed: %v", s.config.Scanning.ScanBinary, args, err)
 		}
 		zs.Errorf("Command output: %s", bytes.TrimSpace(output))
 		if s.config.Scanning.KeepFailedWfps {
 			s.copyWfpTempFile(tempFile.Name(), zs)
 		}
-		return "", fmt.Errorf("failed to scan WFP: %v", err)
+		return "", timeoutEncountered, fmt.Errorf("failed to scan WFP: %v", err)
 	}
-	return string(output), err
+	return string(output), timeoutEncountered, err
 }
 
 // TestEngine tests if the SCANOSS engine is accessible and running.

pkg/service/scanning_service_test.go

@@ -112,17 +112,21 @@ func TestScanDirectSingle(t *testing.T) {
 	myConfig := setupConfig(t)
 	myConfig.App.Trace = true
 	myConfig.Scanning.ScanDebug = true
+	myConfig.Scanning.MatchConfigAllowed = false
+	myConfig.Scanning.RankingAllowed = false
 	apiService := NewAPIService(myConfig)
 
 	tests := []struct {
-		name      string
-		fieldName string
-		file      string
-		binary    string
-		telemetry bool
-		scanType  string
-		assets    string
-		want      int
+		name            string
+		fieldName       string
+		file            string
+		binary          string
+		telemetry       bool
+		scanType        string
+		assets          string
+		scanSettingsB64 string
+		settingsAllowed bool
+		want            int
 	}{
 		{
 			name:      "Scanning - wrong name",
@@ -186,6 +190,77 @@ func TestScanDirectSingle(t *testing.T) {
 			assets:    "pkg:github/org/repo",
 			want:      http.StatusOK,
 		},
+		{
+			name:            "Scanning - Settings - invalid base64",
+			binary:          "../../test-support/scanoss.sh",
+			fieldName:       "filename",
+			file:            "./tests/fingers.wfp",
+			settingsAllowed: true,
+			scanSettingsB64: "invalid-base64!!!",
+			want:            http.StatusBadRequest,
+		},
+		{
+			name:      "Scanning - Settings - invalid json",
+			binary:    "../../test-support/scanoss.sh",
+			fieldName: "filename",
+			file:      "./tests/fingers.wfp",
+			// Base64 decoded JSON:
+			// {
+			//   "field": "something,
+			//   "array": [
+			// }
+			scanSettingsB64: "ewoiZmllbGQiOiAic29tZXRoaW5nLAogImFycmF5IjogWwp9",
+			settingsAllowed: true,
+			want:            http.StatusBadRequest,
+		},
+		{
+			name:      "Scanning - Settings - not allowed",
+			binary:    "../../test-support/scanoss.sh",
+			fieldName: "filename",
+			file:      "./tests/fingers.wfp",
+			// Base64 decoded JSON:
+			// {
+			//   "min_snippet_hits": 5,
+			//   "min_snippet_lines": 10
+			// }
+			scanSettingsB64: "eyJtaW5fc25pcHBldF9oaXRzIjo1LCJtaW5fc25pcHBldF9saW5lcyI6MTB9",
+			settingsAllowed: false,
+			want:            http.StatusBadRequest,
+		},
+		{
+			name:      "Scanning - Settings - success 1",
+			binary:    "../../test-support/scanoss.sh",
+			fieldName: "filename",
+			file:      "./tests/fingers.wfp",
+			// Base64 decoded JSON:
+			// {
+			//   "ranking_enabled": true,
+			//   "ranking_threshold": 85,
+			//   "min_snippet_hits": 3,
+			//   "min_snippet_lines": 8,
+			//   "honour_file_exts": false
+			// }
+			scanSettingsB64: "eyJyYW5raW5nX2VuYWJsZWQiOnRydWUsInJhbmtpbmdfdGhyZXNob2xkIjo4NSwibWluX3NuaXBwZXRfaGl0cyI6MywibWluX3NuaXBwZXRfbGluZXMiOjgsImhvbm91cl9maWxlX2V4dHMiOmZhbHNlfQ==",
+			settingsAllowed: true,
+			want:            http.StatusOK,
+		},
+		{
+			name:      "Scanning - Settings - success 2",
+			binary:    "../../test-support/scanoss.sh",
+			fieldName: "filename",
+			file:      "./tests/fingers.wfp",
+			// Base64 decoded JSON:
+			// {
+			//   "ranking_enabled": true,
+			//   "ranking_threshold": -1,
+			//   "min_snippet_hits": 3,
+			//   "min_snippet_lines": 8,
+			//   "honour_file_exts": true
+			// }
+			scanSettingsB64: "ewogICJyYW5raW5nX2VuYWJsZWQiOiB0cnVlLAogICJyYW5raW5nX3RocmVzaG9sZCI6IC0xLAogICJtaW5fc25pcHBldF9oaXRzIjogMywKICAibWluX3NuaXBwZXRfbGluZXMiOiA4LAogICJob25vdXJfZmlsZV9leHRzIjogdHJ1ZQp9",
+			settingsAllowed: true,
+			want:            http.StatusOK,
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -196,6 +271,8 @@ func TestScanDirectSingle(t *testing.T) {
 					myConfig.App.Trace = true
 				}
 			}
+			myConfig.Scanning.MatchConfigAllowed = test.settingsAllowed
+			myConfig.Scanning.RankingEnabled = test.settingsAllowed
 			myConfig.Scanning.ScanBinary = test.binary
 			myConfig.Telemetry.Enabled = test.telemetry
 			filePath := test.file
@@ -225,9 +302,11 @@ func TestScanDirectSingle(t *testing.T) {
 				}
 			}
 			_ = mw.Close() // close the writer before making the request
-
 			req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody)
 			w := httptest.NewRecorder()
+			if len(test.scanSettingsB64) > 0 {
+				req.Header.Set("Scanoss-Settings", test.scanSettingsB64)
+			}
 			req.Header.Add("Content-Type", mw.FormDataContentType())
 			apiService.ScanDirect(w, req)
 			resp := w.Result()
@@ -449,3 +528,76 @@ func TestScanDirectSingleHPSM(t *testing.T) {
 		})
 	}
 }
+
+func TestScanDirectSingleSlow(t *testing.T) {
+	err := zlog.NewSugaredDevLogger()
+	if err != nil {
+		t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
+	}
+	defer zlog.SyncZap()
+	myConfig := setupConfig(t)
+	myConfig.App.Trace = true
+	myConfig.Scanning.ScanDebug = true
+	myConfig.Scanning.ScanTimeout = 5
+	apiService := NewAPIService(myConfig)
+
+	tests := []struct {
+		name      string
+		fieldName string
+		file      string
+		binary    string
+		scanType  string
+		assets    string
+		want      int
+	}{
+		{
+			name:      "Scanning - success 1",
+			binary:    "../../test-support/scanoss.sh",
+			fieldName: "file",
+			file:      "./tests/fingers.wfp",
+			want:      http.StatusOK,
+		},
+		{
+			name:      "Scanning - Slow fail",
+			binary:    "../../test-support/scanoss-slow.sh",
+			fieldName: "filename",
+			file:      "./tests/fingers-hpsm.wfp",
+			want:      http.StatusGatewayTimeout,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			myConfig.Scanning.ScanBinary = test.binary
+			filePath := test.file
+			fieldName := test.fieldName
+			postBody := new(bytes.Buffer)
+			mw := multipart.NewWriter(postBody)
+			file, err := os.Open(filePath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			writer, err := mw.CreateFormFile(fieldName, filePath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err = io.Copy(writer, file); err != nil {
+				t.Fatal(err)
+			}
+			_ = mw.Close() // close the writer before making the request
+
+			req := httptest.NewRequest(http.MethodPost, "http://localhost/scan/direct", postBody)
+			w := httptest.NewRecorder()
+			req.Header.Add("Content-Type", mw.FormDataContentType())
+			apiService.ScanDirect(w, req)
+			resp := w.Result()
+			body, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("an error was not expected when reading from request: %v", err)
+			}
+			assert.Equal(t, test.want, resp.StatusCode)
+			fmt.Println("Status: ", resp.StatusCode)
+			fmt.Println("Type: ", resp.Header.Get("Content-Type"))
+			fmt.Println("Body: ", string(body))
+		})
+	}
+}

test-support/scanoss-slow.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+###
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Copyright (C) 2018-2023 SCANOSS.COM
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+###
+d=$(dirname "$0")
+# Simulate getting file contents
+if [ "$1" == "-h" ] || [ "$2" == "-h" ] || [ "$1" == "-help" ] || [ "$2" == "-help" ] ; then
+  echo "SCANOSS slow engine simulator help"
+  echo " command options..."
+  exit 0
+fi
+export DELAY=10
+echo "Info: Running slow simulation delay: $DELAY" >&2
+"$d"/scanoss.sh "$@"
+EXIT_CODE=$?
+# Only sleep if the command finished successfully
+if [ $EXIT_CODE -eq 0 ] ; then
+  sleep $DELAY
+fi
+exit $EXIT_CODE