Skip to content

Commit 8cd8441

Browse files
dobracdobrac
committed
refactor(storage): collapse config/resolution into storage_url.go
Move everything about "how a role's config becomes a provider" into one file: Provider consts, StorageSpec, StorageConfig + role configs + With* helpers, ResolveSpec, GetStorageProvider, ParseStorageURL and the legacy env shim. storage.go keeps only the provider-agnostic surface (StorageProvider/Blob/Seekable interfaces, upload helpers). The file header documents the single resolution pipeline and marks the legacy-specific pieces (legacyStorageURL + the env-name fields) as the unit to delete when the legacy envs are retired.
1 parent 421dfa8 commit 8cd8441

2 files changed

Lines changed: 140 additions & 127 deletions

File tree

‎packages/shared/pkg/storage/storage.go‎

Lines changed: 2 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,12 @@ import (
88
"hash"
99
"io"
1010
"os"
11-
"strings"
1211
"time"
1312

1413
"go.opentelemetry.io/otel"
1514
"go.opentelemetry.io/otel/codes"
1615
"go.opentelemetry.io/otel/trace"
1716

18-
"github.com/e2b-dev/infra/packages/shared/pkg/limit"
1917
"github.com/e2b-dev/infra/packages/shared/pkg/storage/storageopts"
2018
)
2119

@@ -40,20 +38,8 @@ var ErrMetadataUnsupported = errors.New("blob does not support reading custom me
4038
// ObjectMetadataSoftDeleted is the storage-index soft-delete tombstone key.
4139
const ObjectMetadataSoftDeleted = storageopts.ObjectMetadataSoftDeleted
4240

43-
type Provider string
44-
45-
const (
46-
GCPStorageProvider Provider = "GCPBucket"
47-
AWSStorageProvider Provider = "AWSBucket"
48-
LocalStorageProvider Provider = "Local"
49-
50-
DefaultStorageProvider Provider = GCPStorageProvider
51-
52-
storageProviderEnv = "STORAGE_PROVIDER"
53-
54-
// MemoryChunkSize must always be bigger or equal to the block size.
55-
MemoryChunkSize = 4 * 1024 * 1024 // 4 MB
56-
)
41+
// MemoryChunkSize must always be bigger or equal to the block size.
42+
const MemoryChunkSize = 4 * 1024 * 1024 // 4 MB
5743

5844
type SeekableObjectType int
5945

@@ -242,97 +228,6 @@ func (e *PeerTransitionedError) Error() string {
242228
return "peer upload completed, reload header from storage"
243229
}
244230

245-
// StorageConfig describes one storage role (templates, build cache): which
246-
// environment variable carries the role's storage URL, and which legacy
247-
// environment variables configure it when no URL is set. All environment
248-
// variables are read lazily at resolve time, so runtime overrides (os.Setenv,
249-
// t.Setenv in tests) are respected.
250-
type StorageConfig struct {
251-
// name identifies the role in error messages.
252-
name string
253-
// storageURLEnv holds the role's storage URL (see ParseStorageURL),
254-
// e.g. "gs://bucket" or "s3://bucket?endpoint=…&s3ForcePathStyle=true".
255-
// When set it is authoritative; otherwise the legacy envs below apply.
256-
storageURLEnv string
257-
// bucketEnv (cloud) and basePathEnv/basePathDefault (local) are the
258-
// legacy env vars consumed by legacyStorageURL.
259-
bucketEnv string
260-
basePathEnv string
261-
basePathDefault string
262-
263-
limiter *limit.Limiter
264-
uploadBaseURL string
265-
hmacKey []byte
266-
}
267-
268-
// ResolveSpec resolves the storage destination for this config. A defined
269-
// storage URL is authoritative; otherwise the legacy environment variables
270-
// (STORAGE_PROVIDER + the role's bucket/base-path envs) are converted into a
271-
// storage URL, so both styles share one parsing and validation path.
272-
func (c StorageConfig) ResolveSpec() (StorageSpec, error) {
273-
raw := strings.TrimSpace(os.Getenv(c.storageURLEnv))
274-
if raw == "" {
275-
legacy, err := legacyStorageURL(c)
276-
if err != nil {
277-
return StorageSpec{}, err
278-
}
279-
raw = legacy
280-
}
281-
282-
return ParseStorageURL(raw)
283-
}
284-
285-
// WithLimiter returns a copy of the config with the given limiter set.
286-
func (c StorageConfig) WithLimiter(limiter *limit.Limiter) StorageConfig {
287-
c.limiter = limiter
288-
289-
return c
290-
}
291-
292-
// WithLocalUpload returns a copy of the config with the given local upload
293-
// parameters set. These are only used when STORAGE_PROVIDER=Local to let the
294-
// filesystem storage provider generate signed URLs for file uploads.
295-
func (c StorageConfig) WithLocalUpload(uploadBaseURL string, hmacKey []byte) StorageConfig {
296-
c.uploadBaseURL = uploadBaseURL
297-
c.hmacKey = hmacKey
298-
299-
return c
300-
}
301-
302-
var TemplateStorageConfig = StorageConfig{
303-
name: "template",
304-
storageURLEnv: "TEMPLATE_STORAGE_URL",
305-
bucketEnv: "TEMPLATE_BUCKET_NAME",
306-
basePathEnv: "LOCAL_TEMPLATE_STORAGE_BASE_PATH",
307-
basePathDefault: "/tmp/templates",
308-
}
309-
310-
var BuildCacheStorageConfig = StorageConfig{
311-
name: "build cache",
312-
storageURLEnv: "BUILD_CACHE_STORAGE_URL",
313-
bucketEnv: "BUILD_CACHE_BUCKET_NAME",
314-
basePathEnv: "LOCAL_BUILD_CACHE_STORAGE_BASE_PATH",
315-
basePathDefault: "/tmp/build-cache",
316-
}
317-
318-
func GetStorageProvider(ctx context.Context, cfg StorageConfig) (StorageProvider, error) {
319-
spec, err := cfg.ResolveSpec()
320-
if err != nil {
321-
return nil, err
322-
}
323-
324-
switch spec.Provider {
325-
case LocalStorageProvider:
326-
return newFileSystemStorage(spec.BasePath, cfg), nil
327-
case AWSStorageProvider:
328-
return newAWSStorage(ctx, spec, cfg.limiter)
329-
case GCPStorageProvider:
330-
return NewGCP(ctx, spec.Bucket, cfg.limiter)
331-
}
332-
333-
return nil, fmt.Errorf("unknown storage provider: %s", spec.Provider)
334-
}
335-
336231
func recordError(span trace.Span, err error) {
337232
if ignoreEOF(err) == nil {
338233
return

‎packages/shared/pkg/storage/storage_url.go‎

Lines changed: 138 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,46 @@
1+
// Storage configuration and resolution: which provider/bucket/connection each
2+
// storage role (templates, build cache) uses.
3+
//
4+
// Every role resolves through a single pipeline:
5+
//
6+
// storage URL (authoritative, e.g. TEMPLATE_STORAGE_URL)
7+
// │ else: legacy envs → legacyStorageURL()
8+
// ▼
9+
// ParseStorageURL() → StorageSpec → GetStorageProvider()
10+
//
11+
// The legacy environment style (STORAGE_PROVIDER + *_BUCKET_NAME /
12+
// LOCAL_*_STORAGE_BASE_PATH + S3_USE_PATH_STYLE) is kept working by converting
13+
// it into the equivalent storage URL; everything legacy-specific is contained
14+
// in legacyStorageURL and the StorageConfig env-name fields, and is deleted
15+
// wholesale once the legacy envs are retired.
16+
117
package storage
218

319
import (
20+
"context"
421
"fmt"
522
"net/url"
623
"os"
724
"strconv"
825
"strings"
926

1027
"github.com/e2b-dev/infra/packages/shared/pkg/env"
28+
"github.com/e2b-dev/infra/packages/shared/pkg/limit"
1129
"github.com/e2b-dev/infra/packages/shared/pkg/utils"
1230
)
1331

32+
type Provider string
33+
34+
const (
35+
GCPStorageProvider Provider = "GCPBucket"
36+
AWSStorageProvider Provider = "AWSBucket"
37+
LocalStorageProvider Provider = "Local"
38+
39+
DefaultStorageProvider Provider = GCPStorageProvider
40+
41+
storageProviderEnv = "STORAGE_PROVIDER"
42+
)
43+
1444
// StorageSpec is a fully resolved storage destination: which provider to use,
1545
// which bucket (or local path) to address, and any per-connection options.
1646
//
@@ -29,14 +59,106 @@ type StorageSpec struct {
2959
// includes the AWS_ENDPOINT_URL environment variable.
3060
Endpoint string
3161
// UsePathStyle forces S3 path-style addressing (https://host/bucket/key).
32-
// Required by most S3-compatible backends. False falls back to the
33-
// S3_USE_PATH_STYLE environment variable.
62+
// Required by most S3-compatible backends.
3463
UsePathStyle bool
3564
// Region overrides the S3 region. Empty means the AWS SDK default
3665
// resolution (AWS_REGION et al.).
3766
Region string
3867
}
3968

69+
// StorageConfig describes one storage role (templates, build cache): which
70+
// environment variable carries the role's storage URL, and which legacy
71+
// environment variables configure it when no URL is set. All environment
72+
// variables are read lazily at resolve time, so runtime overrides (os.Setenv,
73+
// t.Setenv in tests) are respected.
74+
type StorageConfig struct {
75+
// name identifies the role in error messages.
76+
name string
77+
// storageURLEnv holds the role's storage URL (see ParseStorageURL),
78+
// e.g. "gs://bucket" or "s3://bucket?endpoint=…&s3ForcePathStyle=true".
79+
// When set it is authoritative; otherwise the legacy envs below apply.
80+
storageURLEnv string
81+
// bucketEnv (cloud) and basePathEnv/basePathDefault (local) are the
82+
// legacy env vars consumed by legacyStorageURL.
83+
bucketEnv string
84+
basePathEnv string
85+
basePathDefault string
86+
87+
limiter *limit.Limiter
88+
uploadBaseURL string
89+
hmacKey []byte
90+
}
91+
92+
var TemplateStorageConfig = StorageConfig{
93+
name: "template",
94+
storageURLEnv: "TEMPLATE_STORAGE_URL",
95+
bucketEnv: "TEMPLATE_BUCKET_NAME",
96+
basePathEnv: "LOCAL_TEMPLATE_STORAGE_BASE_PATH",
97+
basePathDefault: "/tmp/templates",
98+
}
99+
100+
var BuildCacheStorageConfig = StorageConfig{
101+
name: "build cache",
102+
storageURLEnv: "BUILD_CACHE_STORAGE_URL",
103+
bucketEnv: "BUILD_CACHE_BUCKET_NAME",
104+
basePathEnv: "LOCAL_BUILD_CACHE_STORAGE_BASE_PATH",
105+
basePathDefault: "/tmp/build-cache",
106+
}
107+
108+
// WithLimiter returns a copy of the config with the given limiter set.
109+
func (c StorageConfig) WithLimiter(limiter *limit.Limiter) StorageConfig {
110+
c.limiter = limiter
111+
112+
return c
113+
}
114+
115+
// WithLocalUpload returns a copy of the config with the given local upload
116+
// parameters set. These are only used with the local filesystem provider to
117+
// let it generate signed URLs for file uploads.
118+
func (c StorageConfig) WithLocalUpload(uploadBaseURL string, hmacKey []byte) StorageConfig {
119+
c.uploadBaseURL = uploadBaseURL
120+
c.hmacKey = hmacKey
121+
122+
return c
123+
}
124+
125+
// ResolveSpec resolves the storage destination for this config. A defined
126+
// storage URL is authoritative; otherwise the legacy environment variables
127+
// (STORAGE_PROVIDER + the role's bucket/base-path envs) are converted into a
128+
// storage URL, so both styles share one parsing and validation path.
129+
func (c StorageConfig) ResolveSpec() (StorageSpec, error) {
130+
raw := strings.TrimSpace(os.Getenv(c.storageURLEnv))
131+
if raw == "" {
132+
legacy, err := legacyStorageURL(c)
133+
if err != nil {
134+
return StorageSpec{}, err
135+
}
136+
raw = legacy
137+
}
138+
139+
return ParseStorageURL(raw)
140+
}
141+
142+
// GetStorageProvider resolves the config and constructs the storage provider
143+
// for it.
144+
func GetStorageProvider(ctx context.Context, cfg StorageConfig) (StorageProvider, error) {
145+
spec, err := cfg.ResolveSpec()
146+
if err != nil {
147+
return nil, err
148+
}
149+
150+
switch spec.Provider {
151+
case LocalStorageProvider:
152+
return newFileSystemStorage(spec.BasePath, cfg), nil
153+
case AWSStorageProvider:
154+
return newAWSStorage(ctx, spec, cfg.limiter)
155+
case GCPStorageProvider:
156+
return NewGCP(ctx, spec.Bucket, cfg.limiter)
157+
}
158+
159+
return nil, fmt.Errorf("unknown storage provider: %s", spec.Provider)
160+
}
161+
40162
// ParseStorageURL parses a storage URL into a StorageSpec. The syntax follows
41163
// the gocloud.dev blob URL dialect so the mapping bucket → client/connection is
42164
// declared in one self-describing string:
@@ -52,10 +174,6 @@ type StorageSpec struct {
52174
// fail fast instead of being silently ignored. Credentials are intentionally
53175
// not accepted in URLs; they come from the provider's usual environment
54176
// (ADC / Workload Identity for gs://, AWS_ACCESS_KEY_ID etc. for s3://).
55-
//
56-
// Legacy environment-variable configuration (STORAGE_PROVIDER + bucket/path
57-
// envs) is converted into this URL form by StorageConfig.ResolveSpec, so both
58-
// configuration styles share this single parsing and validation path.
59177
func ParseStorageURL(raw string) (StorageSpec, error) {
60178
u, err := url.Parse(strings.TrimSpace(raw))
61179
if err != nil {
@@ -155,6 +273,20 @@ func parseFileURL(u *url.URL) (StorageSpec, error) {
155273
}, nil
156274
}
157275

276+
func validateBucketURL(u *url.URL) error {
277+
if u.Host == "" {
278+
return fmt.Errorf("storage URL %q: missing bucket name", u)
279+
}
280+
if u.Path != "" && u.Path != "/" {
281+
return fmt.Errorf("storage URL %q: key prefixes are not supported (bucket only)", u)
282+
}
283+
if u.User != nil {
284+
return fmt.Errorf("storage URL %q: credentials in URLs are not supported", u)
285+
}
286+
287+
return nil
288+
}
289+
158290
// legacyStorageURL converts the legacy environment-variable configuration
159291
// (STORAGE_PROVIDER, defaulting to GCPBucket, + the role's bucket/base-path
160292
// envs + S3_USE_PATH_STYLE) into a storage URL, so both configuration styles
@@ -198,17 +330,3 @@ func legacyStorageURL(cfg StorageConfig) (string, error) {
198330
func (c StorageConfig) legacyBucket() string {
199331
return utils.RequiredEnv(c.bucketEnv, fmt.Sprintf("Bucket for storing %s files", c.name))
200332
}
201-
202-
func validateBucketURL(u *url.URL) error {
203-
if u.Host == "" {
204-
return fmt.Errorf("storage URL %q: missing bucket name", u)
205-
}
206-
if u.Path != "" && u.Path != "/" {
207-
return fmt.Errorf("storage URL %q: key prefixes are not supported (bucket only)", u)
208-
}
209-
if u.User != nil {
210-
return fmt.Errorf("storage URL %q: credentials in URLs are not supported", u)
211-
}
212-
213-
return nil
214-
}

0 commit comments

Comments
 (0)