Adding Azure blob support

.gitignore

@@ -12,3 +12,5 @@
 .idea
 /.venv
 /site
+docker-compose-prod.yml
+.netrc

README.md

@@ -112,7 +112,66 @@ To view a dump of the LMDB contents:
 You can browse the snapshots in MinIO at <http://localhost:4731/buckets/lightningstream/browse>
 (login with minioadmin / minioadmin).
 
+## Using Azure Blob storage
 
+Lightning Stream supports Azure Blob Storage as a backend for storing snapshots. You can configure it to use either static credentials (account name and key) or Azure service principal authentication.
+
+### Basic configuration with static credentials
+
+```yaml
+storage:
+  type: azure
+  options:
+    account_name: myaccountname
+    account_key: myaccountkey
+    use_shared_key: true
+    container: lightningstream
+    create_container: true
+```
+
+### Configuration with Azure service principal (recommended for production)
+
+When `account_key` is not set, the backend uses [`DefaultAzureCredential`](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential), which automatically picks up service principal credentials from environment variables:
+- `AZURE_CLIENT_ID`
+- `AZURE_TENANT_ID`
+- `AZURE_CLIENT_SECRET`
+
+```yaml
+storage:
+  type: azure
+  options:
+    container: lightningstream
+    endpoint_url: https://myaccount.blob.core.windows.net/
+    create_container: true
+```
+
+### Available options
+
+| Option | Type | Summary |
+|--------|------|---------|
+| account_name | string | Azure storage account name (required for shared key auth) |
+| account_key | string | Azure storage account key |
+| use_shared_key | bool | Use shared key (account name + key) authentication; if false, `DefaultAzureCredential` is used |
+| container | string | Azure blob container name (required) |
+| create_container | bool | Create container if it does not exist |
+| endpoint_url | string | Custom endpoint URL (defaults to `https://<account_name>.blob.core.windows.net/`) |
+| global_prefix | string | Transparently apply a global prefix to all blob names |
+| disable_send_content_md5 | bool | Disable sending the Content-MD5 header |
+| tls | [tlsconfig.Config](https://github.com/PowerDNS/go-tlsconfig) | TLS configuration |
+| init_timeout | duration | Time allowed for initialisation (default: "20s") |
+| use_update_marker | bool | Reduce LIST commands by using an update marker (see below) |
+| update_marker_force_list_interval | duration | Force full LIST sync at this interval (default: "5m") |
+| concurrency | int | Max number of concurrent uploads (default: 1) |
+
+The `use_update_marker` option can reduce Azure costs by replacing LIST operations (which are more expensive) with GET operations. However, it cannot be used if the container itself is replicated in an active-active fashion between data centers.
+
+You can see a working example in the docker-compose setup, which uses [Azurite](https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite) (Azure Storage Emulator):
+
+```bash
+docker-compose up
+```
+
+For all available Azure backend options with full descriptions, see [Simpleblob's Azure backend Options struct](https://github.com/PowerDNS/simpleblob/blob/main/backends/azure/azure.go).
 
 ## Open Source
 
@@ -123,5 +182,3 @@ For more information on how we provide support for Open Source products, please
 PowerDNS also offers an Enterprise edition of Lightning Stream that includes professional support, advanced features, deployment
 tooling for large deployments, Kubernetes integration, and more.
 
-
-

cmd/lightningstream/main.go

@@ -4,6 +4,7 @@ import (
 	"github.com/PowerDNS/lightningstream/cmd/lightningstream/commands"
 
 	// Register storage backends
+	_ "github.com/PowerDNS/simpleblob/backends/azure"
 	_ "github.com/PowerDNS/simpleblob/backends/fs"
 	_ "github.com/PowerDNS/simpleblob/backends/memory"
 	_ "github.com/PowerDNS/simpleblob/backends/s3"

config/config.go

@@ -302,7 +302,7 @@ type DBIOptions struct {
 }
 
 type Storage struct {
-	Type    string                 `yaml:"type"`    // "fs", "s3", "memory"
+	Type    string                 `yaml:"type"`    // "fs", "s3", "memory", "azure"
 	Options map[string]interface{} `yaml:"options"` // backend specific
 
 	// FIXME: Configure per LMDB instead, since we run a cleaner per LMDB?

docker-compose.yml

@@ -6,6 +6,7 @@ version: '2.4'
 volumes:
     lmdb:
         driver: local
+    azurite_data:
     minio:
         driver: local
     #snapshots:
@@ -21,13 +22,28 @@ services:
         command: server /data --console-address :9001
         volumes:
             - "minio:/data" 
-
+    # https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite
+    azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        container_name: azurite
+        hostname: azurite
+        restart: always
+        ports:
+            - "${DEVENV_BIND_IP:-127.0.0.1}:10000:10000"
+            - "${DEVENV_BIND_IP:-127.0.0.1}:10001:10001"
+            - "${DEVENV_BIND_IP:-127.0.0.1}:10002:10002"
+        command: azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --silent
+        volumes:    
+        - azurite_data:/data
+        - ./azure/certs:/certs
+        environment:
+         - AZURITE_ACCOUNTS=devstoreaccount1:key1
     auth1:
         image: powerdns/pdns-auth-49
         environment:
-            instance: 1
-            port: 53
-            webserver_port: 8081
+            - instance=1
+            - port=53
+            - webserver_port=8081
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}51:53/tcp"
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}51:53/udp"
@@ -42,9 +58,9 @@ services:
     auth2:
         image: powerdns/pdns-auth-49
         environment:
-            instance: 2
-            port: 53
-            webserver_port: 8081
+            - instance=2
+            - port=53
+            - webserver_port=8081
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}52:53/tcp"
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}52:53/udp"
@@ -59,9 +75,9 @@ services:
     auth3:
         image: powerdns/pdns-auth-49
         environment:
-            instance: 3
-            port: 53
-            webserver_port: 8081
+            - instance=3
+            - port=53
+            - webserver_port=8081
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}53:53/tcp"
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}53:53/udp"
@@ -78,7 +94,7 @@ services:
             dockerfile: Dockerfile
             context: .
         environment:
-            instance: 1  # used in config file
+            - instance=1  # used in config file
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}91:8500"
         volumes:
@@ -94,7 +110,7 @@ services:
             dockerfile: Dockerfile
             context: .
         environment:
-            instance: 2  # used in config file
+            - instance=2  # used in config file
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}92:8500"
         volumes:
@@ -110,13 +126,12 @@ services:
             dockerfile: Dockerfile
             context: .
         environment:
-            instance: 3  # used in config file
+            - instance=3  # used in config file
         ports:
             - "${DEVENV_BIND_IP:-127.0.0.1}:${PORT_PREFIX:-47}93:8500"
         volumes:
             - "lmdb:/lmdb" 
             - "./docker/pdns/lightningstream.yaml:/lightningstream.yaml:ro"
-            #- "snapshots:/snapshots" 
         working_dir: /
         user: "953"  # pdns
         command: --minimum-pid 50 receive

docker/pdns/lightningstream.yaml

@@ -26,20 +26,31 @@ sweeper:
 
 storage:
   #type: fs
-  type: s3
-  options:
-    #root_path: /snapshots
-    access_key: minioadmin
-    secret_key: minioadmin
-    region: us-east-1
-    bucket: lightningstream
-    endpoint_url: http://minio:9000
-    create_bucket: true
+  # type: s3
+  # options:
+  #   #root_path: /snapshots
+  #   access_key: minioadmin
+  #   secret_key: minioadmin
+  #   region: us-east-1
+  #   bucket: lightningstream
+  #   endpoint_url: http://minio:9000
+  #   create_bucket: true
   cleanup:
     enabled: true
     interval: 15m
     must_keep_interval: 24h
     remove_old_instances_interval: 168h
+  type: azure
+  options:
+    account_name: devstoreaccount1
+    account_key: key1
+    container: lightningstreamcontainer
+    endpoint_url: http://azurite:10000/devstoreaccount1
+    create_container: true
+    use_update_marker: false
+    use_shared_key: true
+
+storage_poll_interval: 5s
 
 http:
   address: ":8500"

docs/commands.md

@@ -361,6 +361,7 @@ lightningstream sync [flags]
 
 ```
   -h, --help                          help for sync
+      --only-db stringArray           Only sync this named db (can be repeated)
       --only-once                     Only do a single run and exit
       --wait-for-marker-file string   Marker file to wait for in storage before starting syncers
 ```