chore(storage): storage file structure changes

Dockerfile

@@ -77,7 +77,7 @@
     org.opencontainers.image.source="https://github.com/qdm12/gluetun" \
     org.opencontainers.image.title="VPN swiss-knife like client for multiple VPN providers" \
     org.opencontainers.image.description="VPN swiss-knife like client to tunnel to multiple VPN servers using OpenVPN, IPtables, DNS over TLS, Shadowsocks, an HTTP proxy and Alpine Linux"
 ENV VPN_SERVICE_PROVIDER=pia \
     VPN_TYPE=openvpn \
     # Common VPN options
     VPN_INTERFACE=tun0 \
@@ -249,6 +249,7 @@
     UPDATER_PERIOD=0 \
     UPDATER_MIN_RATIO=0.8 \
     UPDATER_VPN_SERVICE_PROVIDERS= \
+    UPDATER_PREFER_DIRECT_DOWNLOAD=no \
     UPDATER_PROTONVPN_EMAIL= \
     UPDATER_PROTONVPN_PASSWORD= \
     # Public IP
@@ -257,7 +258,8 @@
     PUBLICIP_API=ipinfo,ifconfigco,ip2location,cloudflare \
     PUBLICIP_API_TOKEN= \
     # Storage
-    STORAGE_FILEPATH=/gluetun/servers.json \
+    STORAGE_SERVERS_ENABLED=on \
+    STORAGE_SERVERS_DIRECTORY_PATH=/gluetun/servers/ \
     # Pprof
     PPROF_ENABLED=no \
     PPROF_BLOCK_PROFILE_RATE=0 \

cmd/gluetun/main.go

@@ -169,21 +169,19 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 
 	defer fmt.Println(gluetunLogo)
 
-	announcementExp, err := time.Parse(time.RFC3339, "2026-04-30T00:00:00Z")
+	announcementExp, err := time.Parse(time.RFC3339, "2026-06-30T00:00:00Z")
 	if err != nil {
 		return err
 	}
 	splashSettings := gosplash.Settings{
-		User:       "qdm12",
-		Repository: "gluetun",
-		Emails:     []string{"quentin.mcgaw@gmail.com"},
-		Version:    buildInfo.Version,
-		Commit:     buildInfo.Commit,
-		Created:    buildInfo.Created,
-		Announcement: "the repository will be migrated to https://github.com/passteque/gluetun on 2026-05-21, " +
-			"which is a Github organization under my sole control, so don't get alarmed if you get redirected " +
-			"in the coming days (reason: personal paperwork ugh)",
-		AnnounceExp: announcementExp,
+		User:         "qdm12",
+		Repository:   "gluetun",
+		Emails:       []string{"quentin.mcgaw@gmail.com"},
+		Version:      buildInfo.Version,
+		Commit:       buildInfo.Commit,
+		Created:      buildInfo.Created,
+		Announcement: "Your servers data files are now migrated to /gluetun/servers/",
+		AnnounceExp:  announcementExp,
 		// Sponsor information
 		PaypalUser:    "qmcgaw",
 		GithubSponsor: "qdm12",
@@ -245,7 +243,8 @@ func _main(ctx context.Context, buildInfo models.BuildInformation,
 
 	// TODO run this in a loop or in openvpn to reload from file without restarting
 	storageLogger := logger.New(log.SetComponent("storage"))
-	storage, err := storage.New(storageLogger, *allSettings.Storage.Filepath)
+	storage, err := storage.New(storageLogger, *allSettings.Storage.ServersEnabled,
+		allSettings.Storage.ServersPath, allSettings.Storage.LegacyServersFilepath)
 	if err != nil {
 		return err
 	}

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/mdlayher/netlink v1.9.0
 	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a
+	github.com/qdm12/gluetun-servers v0.1.0
 	github.com/qdm12/gosettings v0.4.4
 	github.com/qdm12/goshutdown v0.3.0
 	github.com/qdm12/gosplash v0.2.1-0.20260305164749-b713de4fee6c

go.sum

@@ -76,6 +76,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a h1:TE157yPQmAbVruH0MWCQzs0vTT/6t96DkoWUXd6PVuc=
 github.com/qdm12/dns/v2 v2.0.0-rc9.0.20260421173011-9de8e7fdbe3a/go.mod h1:98foWgXJZ+g8gJIuO+fdO+oWpFei5WShMFTeN4Im2lE=
+github.com/qdm12/gluetun-servers v0.1.0 h1:w9JLghKZwI0Gzpp9p5rNANgEYUUZ1dxdxsG6NKIojaY=
+github.com/qdm12/gluetun-servers v0.1.0/go.mod h1:acttuyHyoFDu6GTbf3kAV+QXeiX8oJeh0MBic67/9z8=
 github.com/qdm12/goservices v0.1.1-0.20251104135713-6bee97bd4978 h1:TRGpCU1l0lNwtogEUSs5U+RFceYxkAJUmrGabno7J5c=
 github.com/qdm12/goservices v0.1.1-0.20251104135713-6bee97bd4978/go.mod h1:D1Po4CRQLYjccnAR2JsVlN1sBMgQrcNLONbvyuzcdTg=
 github.com/qdm12/gosettings v0.4.4 h1:SM6tOZDf6k8qbjWU8KWyBF4mWIixfsKCfh9DGRLHlj4=

internal/cli/cli.go

@@ -1,11 +1,7 @@
 package cli
 
-type CLI struct {
-	repoServersPath string
-}
+type CLI struct{}
 
 func New() *CLI {
-	return &CLI{
-		repoServersPath: "./internal/storage/servers.json",
-	}
+	return &CLI{}
 }

internal/cli/formatservers.go

@@ -9,9 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/constants/providers"
-	"github.com/qdm12/gluetun/internal/storage"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
@@ -74,10 +72,9 @@ func (c *CLI) FormatServers(args []string) error {
 		}
 	}
 
-	logger := newNoopLogger()
-	storage, err := storage.New(logger, constants.ServersData)
+	storage, err := setupStorage(newNoopLogger())
 	if err != nil {
-		return fmt.Errorf("creating servers storage: %w", err)
+		return fmt.Errorf("setting up storage: %w", err)
 	}
 
 	formatted, err := storage.Format(providerToFormat, format)

internal/cli/helpers.go

@@ -0,0 +1,39 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/qdm12/gluetun/internal/configuration/settings"
+	"github.com/qdm12/gluetun/internal/configuration/sources/files"
+	"github.com/qdm12/gluetun/internal/configuration/sources/secrets"
+	"github.com/qdm12/gluetun/internal/storage"
+	"github.com/qdm12/gosettings/reader"
+	"github.com/qdm12/gosettings/reader/sources/env"
+)
+
+type storageSetupLogger interface {
+	storage.Logger
+	files.Warner
+}
+
+func setupStorage(logger storageSetupLogger) (s *storage.Storage, err error) {
+	settingsReader := reader.New(reader.Settings{
+		Sources: []reader.Source{
+			secrets.New(logger),
+			files.New(logger),
+			env.New(env.Settings{}),
+		},
+	})
+	var settings settings.Storage
+	err = settings.Read(settingsReader)
+	if err != nil {
+		return nil, fmt.Errorf("reading storage settings: %w", err)
+	}
+	settings.SetDefaults()
+	storage, err := storage.New(logger, *settings.ServersEnabled, settings.ServersPath,
+		settings.LegacyServersFilepath)
+	if err != nil {
+		return nil, fmt.Errorf("creating storage: %w", err)
+	}
+	return storage, nil
+}

internal/cli/nooplogger.go

@@ -6,5 +6,7 @@ func newNoopLogger() *noopLogger {
 	return new(noopLogger)
 }
 
-func (l *noopLogger) Info(string) {}
-func (l *noopLogger) Warn(string) {}
+func (l *noopLogger) Info(string)          {}
+func (l *noopLogger) Infof(string, ...any) {}
+func (l *noopLogger) Warn(string)          {}
+func (l *noopLogger) Warnf(string, ...any) {}

internal/cli/openvpnconfig.go

@@ -9,12 +9,10 @@ import (
 	"time"
 
 	"github.com/qdm12/gluetun/internal/configuration/settings"
-	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/models"
 	"github.com/qdm12/gluetun/internal/netlink"
 	"github.com/qdm12/gluetun/internal/openvpn/extract"
 	"github.com/qdm12/gluetun/internal/provider"
-	"github.com/qdm12/gluetun/internal/storage"
 	"github.com/qdm12/gluetun/internal/updater/resolver"
 	"github.com/qdm12/gosettings/reader"
 )
@@ -49,9 +47,9 @@ type IPv6Checker interface {
 func (c *CLI) OpenvpnConfig(logger OpenvpnConfigLogger, reader *reader.Reader,
 	ipv6Checker IPv6Checker,
 ) error {
-	storage, err := storage.New(logger, constants.ServersData)
+	storage, err := setupStorage(newNoopLogger())
 	if err != nil {
-		return err
+		return fmt.Errorf("setting up storage: %w", err)
 	}
 
 	var allSettings settings.Settings

internal/cli/update.go

@@ -13,31 +13,30 @@ import (
 	"github.com/qdm12/dns/v2/pkg/doh"
 	dnsprovider "github.com/qdm12/dns/v2/pkg/provider"
 	"github.com/qdm12/gluetun/internal/configuration/settings"
-	"github.com/qdm12/gluetun/internal/constants"
 	"github.com/qdm12/gluetun/internal/constants/providers"
 	"github.com/qdm12/gluetun/internal/openvpn/extract"
 	"github.com/qdm12/gluetun/internal/provider"
 	"github.com/qdm12/gluetun/internal/publicip/api"
-	"github.com/qdm12/gluetun/internal/storage"
 	"github.com/qdm12/gluetun/internal/updater"
 	"github.com/qdm12/gluetun/internal/updater/resolver"
 	"github.com/qdm12/gluetun/internal/updater/unzip"
 )
 
 type UpdaterLogger interface {
 	Info(s string)
+	Infof(format string, args ...any)
 	Warn(s string)
+	Warnf(format string, args ...any)
 	Error(s string)
 }
 
 func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) error {
 	options := settings.Updater{}
-	var endUserMode, maintainerMode, updateAll bool
+	// TODO v4: remove flags below already present in standard settings
+	var endUserMode, maintainerMode bool
+	var updateAll bool
 	var dnsServer, csvProviders, ipToken, protonUsername, protonEmail, protonPassword string
 	flagSet := flag.NewFlagSet("update", flag.ExitOnError)
-	flagSet.BoolVar(&endUserMode, "enduser", false, "Write results to /gluetun/servers.json (for end users)")
-	flagSet.BoolVar(&maintainerMode, "maintainer", false,
-		"Write results to ./internal/storage/servers.json to modify the program (for maintainers)")
 	flagSet.StringVar(&dnsServer, "dns", "", "no longer used, your DNS will use DoH with Cloudflare and Google")
 	const defaultMinRatio = 0.8
 	flagSet.Float64Var(&options.MinRatio, "minratio", defaultMinRatio,
@@ -49,16 +48,19 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 		"(Retro-compatibility) Username to use to authenticate with Proton. Use -proton-email instead.") // v4 remove this
 	flagSet.StringVar(&protonEmail, "proton-email", "", "Email to use to authenticate with Proton")
 	flagSet.StringVar(&protonPassword, "proton-password", "", "Password to use to authenticate with Proton")
+	flagSet.BoolVar(&endUserMode, "enduser", false, "deprecated")
+	flagSet.BoolVar(&maintainerMode, "maintainer", false, "deprecated")
 	if err := flagSet.Parse(args); err != nil {
 		return err
 	}
 
-	if dnsServer != "" {
+	switch {
+	case dnsServer != "":
 		logger.Warn("The -dns flag is no longer used, your DNS will use DoH with Cloudflare and Google")
-	}
-
-	if !endUserMode && !maintainerMode {
-		return errors.New("at least one of -enduser or -maintainer must be specified")
+	case endUserMode:
+		logger.Warn("The -enduser flag is now unused")
+	case maintainerMode:
+		logger.Warn("The -maintainer flag is now unused")
 	}
 
 	if updateAll {
@@ -87,11 +89,7 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 		return fmt.Errorf("options validation failed: %w", err)
 	}
 
-	serversDataPath := constants.ServersData
-	if maintainerMode {
-		serversDataPath = ""
-	}
-	storage, err := storage.New(logger, serversDataPath)
+	storage, err := setupStorage(logger)
 	if err != nil {
 		return fmt.Errorf("creating servers storage: %w", err)
 	}
@@ -127,18 +125,11 @@ func (c *CLI) Update(ctx context.Context, args []string, logger UpdaterLogger) e
 	providers := provider.NewProviders(storage, time.Now, logger, httpClient,
 		unzipper, parallelResolver, ipFetcher, openvpnFileExtractor, options)
 
-	updater := updater.New(httpClient, storage, providers, logger)
+	updater := updater.New(httpClient, storage, providers, logger, *options.PreferDirectDownload)
 	err = updater.UpdateServers(ctx, options.Providers, options.MinRatio)
 	if err != nil {
 		return fmt.Errorf("updating server information: %w", err)
 	}
 
-	if maintainerMode {
-		err := storage.FlushToFile(c.repoServersPath)
-		if err != nil {
-			return fmt.Errorf("writing servers data to embedded JSON file: %w", err)
-		}
-	}
-
 	return nil
 }

internal/configuration/settings/settings.go

@@ -132,7 +132,7 @@ func (s *Settings) SetDefaults() {
 	s.IPv6.setDefaults()
 	s.PublicIP.setDefaults()
 	s.Shadowsocks.setDefaults()
-	s.Storage.setDefaults()
+	s.Storage.SetDefaults()
 	s.System.setDefaults()
 	s.Version.setDefaults()
 	s.VPN.setDefaults()
@@ -213,7 +213,7 @@ func (s *Settings) Read(r *reader.Reader, warner Warner) (err error) {
 			return s.PublicIP.read(r, warner)
 		},
 		"shadowsocks": s.Shadowsocks.read,
-		"storage":     s.Storage.read,
+		"storage":     s.Storage.Read,
 		"system":      s.System.read,
 		"updater":     s.Updater.read,
 		"version":     s.Version.read,

internal/configuration/settings/settings_test.go

@@ -90,7 +90,7 @@ func Test_Settings_String(t *testing.T) {
 |   ├── Logging: yes
 |   └── Authentication file path: /gluetun/auth/config.toml
 ├── Storage settings:
-|   └── Filepath: /gluetun/servers.json
+|   └── Servers directory path: /gluetun/servers/
 ├── OS Alpine settings:
 |   ├── Process UID: 1000
 |   └── Process GID: 1000