Add an experiment to disable pip build isolation

sdks/python/container/boot.go

@@ -30,6 +30,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 	"sync"
 	"syscall"
@@ -116,6 +117,37 @@ func main() {
 	}
 }
 
+// The json string of pipeline options is in the following format.
+// We only focus on experiments here.
+//
+//	{
+//		 "display_data": [
+//		  	{...},
+//		 ],
+//		 "options": {
+//		  	...
+//			  "experiments": [
+//				...
+//			 ],
+//		 }
+//	}
+type PipelineOptionsData struct {
+	Options OptionsData `json:"options"`
+}
+
+type OptionsData struct {
+	Experiments []string `json:"experiments"`
+}
+
+func getExperiments(options string) []string {
+	var opts PipelineOptionsData
+	err := json.Unmarshal([]byte(options), &opts)
+	if err != nil {
+		return nil
+	}
+	return opts.Options.Experiments
+}
+
 func launchSDKProcess() error {
 	ctx := grpcx.WriteWorkerID(context.Background(), *id)
 
@@ -155,6 +187,13 @@ func launchSDKProcess() error {
 		logger.Fatalf(ctx, "Failed to convert pipeline options: %v", err)
 	}
 
+	experiments := getExperiments(options)
+	pipNoBuildIsolation = false
+	if slices.Contains(experiments, "pip_no_build_isolation") {
+		pipNoBuildIsolation = true
+		logger.Printf(ctx, "Disabled build isolation when installing packages with pip")
+	}
+
 	// (2) Retrieve and install the staged packages.
 	//
 	// No log.Fatalf() from here on, otherwise deferred cleanups will not be called!

sdks/python/container/piputil.go

@@ -32,6 +32,11 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx"
 )
 
+var (
+	// Whether to append "--no-build-isolation" flag to pip install command
+	pipNoBuildIsolation bool
+)
+
 const pipLogFlushInterval time.Duration = 15 * time.Second
 const unrecoverableURL string = "https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures"
 
@@ -112,6 +117,9 @@ func pipInstallPackage(ctx context.Context, logger *tools.Logger, files []string
 				// installed if necessary.  This achieves our goal outlined above.
 				args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "--upgrade", "--force-reinstall", "--no-deps",
 					filepath.Join(dir, packageSpec)}
+				if pipNoBuildIsolation {
+					args = append(args, "--no-build-isolation")
+				}
 				err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...)
 				if err != nil {
 					bufLogger.FlushAtError(ctx)
@@ -120,6 +128,9 @@ func pipInstallPackage(ctx context.Context, logger *tools.Logger, files []string
 					bufLogger.FlushAtDebug(ctx)
 				}
 				args = []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)}
+				if pipNoBuildIsolation {
+					args = append(args, "--no-build-isolation")
+				}
 				err = execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...)
 				if err != nil {
 					bufLogger.FlushAtError(ctx)
@@ -131,6 +142,9 @@ func pipInstallPackage(ctx context.Context, logger *tools.Logger, files []string
 
 			// Case when we do not perform a forced reinstall.
 			args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)}
+			if pipNoBuildIsolation {
+				args = append(args, "--no-build-isolation")
+			}
 			err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...)
 			if err != nil {
 				bufLogger.FlushAtError(ctx)