feat: support STORAGE_EMULATOR_HOST environment variable

google-cloud-storage/lib/google-cloud-storage.rb

@@ -77,7 +77,8 @@ module Cloud
     #   readonly_storage = gcloud.storage scope: readonly_scope
     #
     def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil,
-                max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil
+                max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil,
+                emulator_host: nil
       Google::Cloud.storage @project, @keyfile, scope: scope,
                                                 retries: retries || @retries,
                                                 timeout: timeout || @timeout,
@@ -88,7 +89,8 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time
                                                 base_interval: base_interval,
                                                 max_interval: max_interval,
                                                 multiplier: multiplier,
-                                                upload_chunk_size: upload_chunk_size
+                                                upload_chunk_size: upload_chunk_size,
+                                                emulator_host: emulator_host
     end
 
     ##
@@ -142,7 +144,7 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time
     def self.storage project_id = nil, credentials = nil, scope: nil,
                      retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil,
                      max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil,
-                     upload_chunk_size: nil
+                     upload_chunk_size: nil, emulator_host: nil
       require "google/cloud/storage"
       Google::Cloud::Storage.new project_id: project_id,
                                  credentials: credentials,
@@ -156,7 +158,8 @@ def self.storage project_id = nil, credentials = nil, scope: nil,
                                  base_interval: base_interval,
                                  max_interval: max_interval,
                                  multiplier: multiplier,
-                                 upload_chunk_size: upload_chunk_size
+                                 upload_chunk_size: upload_chunk_size,
+                                 emulator_host: emulator_host
     end
   end
 end
@@ -172,6 +175,9 @@ def self.storage project_id = nil, credentials = nil, scope: nil,
       "STORAGE_KEYFILE", "STORAGE_KEYFILE_JSON"
     )
   end
+  default_emulator = Google::Cloud::Config.deferred do
+    ENV["STORAGE_EMULATOR_HOST"]
+  end
 
   config.add_field! :project_id, default_project, match: String, allow_nil: true
   config.add_alias! :project, :project_id
@@ -192,5 +198,6 @@ def self.storage project_id = nil, credentials = nil, scope: nil,
   config.add_field! :send_timeout, nil, match: Integer
   config.add_field! :upload_chunk_size, nil, match: Integer
   config.add_field! :endpoint, nil, match: String, allow_nil: true
+  config.add_field! :emulator_host, default_emulator, match: String, allow_nil: true
   config.add_field! :universe_domain, nil, match: String, allow_nil: true
 end

google-cloud-storage/lib/google/cloud/storage.rb

@@ -70,6 +70,11 @@ module Storage
       # @param [Integer] send_timeout How long, in seconds, before receiving response from server times out. Optional.
       # @param [String] endpoint Override of the endpoint host name. Optional.
       #   If the param is nil, uses the default endpoint.
+      # @param [String] emulator_host Address of a Storage emulator to connect
+      #   to, including the scheme (e.g. `http://localhost:9000`). Optional.
+      #   Defaults to the `STORAGE_EMULATOR_HOST` environment variable when set.
+      #   When present, the client connects to the emulator and does not require
+      #   credentials (it connects anonymously unless credentials are provided).
       # @param universe_domain [String] Override of the universe domain. Optional.
       #   If unset or nil, uses the default unvierse domain
       # @param [Integer] upload_chunk_size The chunk size of storage upload, in bytes.
@@ -97,28 +102,36 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil,
                    timeout: nil, open_timeout: nil, read_timeout: nil,
                    send_timeout: nil, endpoint: nil, project: nil, keyfile: nil,
                    max_elapsed_time: nil, base_interval: nil, max_interval: nil,
-                   multiplier: nil, upload_chunk_size: nil, universe_domain: nil
+                   multiplier: nil, upload_chunk_size: nil, universe_domain: nil,
+                   emulator_host: nil
         scope             ||= configure.scope
         retries           ||= configure.retries
         timeout           ||= configure.timeout
         open_timeout      ||= configure.open_timeout || timeout
         read_timeout      ||= configure.read_timeout || timeout
         send_timeout      ||= configure.send_timeout || timeout
         endpoint          ||= configure.endpoint
-        credentials       ||= keyfile || default_credentials(scope: scope)
         max_elapsed_time  ||= configure.max_elapsed_time
         base_interval     ||= configure.base_interval
         max_interval      ||= configure.max_interval
         multiplier        ||= configure.multiplier
         upload_chunk_size ||= configure.upload_chunk_size
         universe_domain   ||= configure.universe_domain
+        emulator_host     ||= configure.emulator_host
 
-        unless credentials.is_a? Google::Auth::Credentials
+        if emulator_host
+          endpoint = emulator_host
+          credentials ||= keyfile
+        else
+          credentials ||= keyfile || default_credentials(scope: scope)
+        end
+
+        if credentials && !credentials.is_a?(Google::Auth::Credentials)
           credentials = Storage::Credentials.new credentials, scope: scope
         end
 
         project_id = resolve_project_id(project_id || project, credentials)
-        raise ArgumentError, "project_id is missing" if project_id.empty?
+        raise ArgumentError, "project_id is missing" if project_id.empty? && !emulator_host
 
         Storage::Project.new(
           Storage::Service.new(
@@ -203,6 +216,10 @@ def self.anonymous retries: nil, timeout: nil, open_timeout: nil,
       #   parameter `keyfile` is considered deprecated, but may also be used.)
       # * `endpoint` - (String) Override of the endpoint host name, or `nil`
       #   to use the default endpoint.
+      # * `emulator_host` - (String) Address of a Storage emulator to connect to,
+      #   including the scheme (e.g. `http://localhost:9000`), or `nil` to connect
+      #   to the live service. Defaults to the `STORAGE_EMULATOR_HOST` environment
+      #   variable when set.
       # * `scope` - (String, Array<String>) The OAuth 2.0 scopes controlling
       #   the set of resources and operations that the connection can access.
       # * `retries` - (Integer) Number of times to retry requests on server

google-cloud-storage/test/google/cloud/storage_test.rb

@@ -19,7 +19,7 @@
   describe "#storage" do
     it "calls out to Google::Cloud.storage" do
       gcloud = Google::Cloud.new
-      stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) {
+      stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) {
         _(project).must_be :nil?
         _(keyfile).must_be :nil?
         _(scope).must_be :nil?
@@ -44,7 +44,7 @@
 
     it "passes project and keyfile to Google::Cloud.storage" do
       gcloud = Google::Cloud.new "project-id", "keyfile-path"
-      stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) {
+      stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) {
         _(project).must_equal "project-id"
         _(keyfile).must_equal "keyfile-path"
         _(scope).must_be :nil?
@@ -69,7 +69,7 @@
 
     it "passes project and keyfile and options to Google::Cloud.storage" do
       gcloud = Google::Cloud.new "project-id", "keyfile-path"
-      stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) {
+      stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) {
         _(project).must_equal "project-id"
         _(keyfile).must_equal "keyfile-path"
         _(scope).must_equal "http://example.com/scope"
@@ -387,6 +387,103 @@ def creds.is_a? target
     end
   end
 
+  describe "Storage.emulator" do
+    let(:emulator_host) { "http://localhost:9000" }
+    let(:default_credentials) do
+      creds = OpenStruct.new empty: true
+      def creds.is_a? target
+        target == Google::Auth::Credentials
+      end
+      creds
+    end
+
+    it "uses STORAGE_EMULATOR_HOST environment variable" do
+      emulator_check = ->(name) { (name == "STORAGE_EMULATOR_HOST") ? emulator_host : nil }
+      stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
+        max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
+        _(project).must_equal "project-id"
+        _(credentials).must_be :nil?
+        _(host).must_equal emulator_host
+        OpenStruct.new project: project
+      }
+
+      # Clear all environment variables, except STORAGE_EMULATOR_HOST
+      ENV.stub :[], emulator_check do
+        # Get project_id from Google Compute Engine
+        Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do
+          Google::Cloud::Storage::Service.stub :new, stubbed_service do
+            storage = Google::Cloud::Storage.new
+            _(storage).must_be_kind_of Google::Cloud::Storage::Project
+            _(storage.project).must_equal "project-id"
+          end
+        end
+      end
+    end
+
+    it "allows emulator_host to be set" do
+      stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
+        max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
+        _(project).must_equal "project-id"
+        _(credentials).must_be :nil?
+        _(host).must_equal emulator_host
+        OpenStruct.new project: project
+      }
+
+      # Clear all environment variables
+      ENV.stub :[], nil do
+        Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do
+          Google::Cloud::Storage::Service.stub :new, stubbed_service do
+            storage = Google::Cloud::Storage.new emulator_host: emulator_host
+            _(storage).must_be_kind_of Google::Cloud::Storage::Project
+            _(storage.project).must_equal "project-id"
+          end
+        end
+      end
+    end
+
+    it "does not require a project_id when using an emulator" do
+      stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
+        max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
+        _(project).must_equal ""
+        _(credentials).must_be :nil?
+        _(host).must_equal emulator_host
+        OpenStruct.new project: project
+      }
+
+      # Clear all environment variables, and provide no project_id from any source
+      ENV.stub :[], nil do
+        Google::Cloud.stub :env, OpenStruct.new(project_id: nil) do
+          Google::Cloud::Storage::Service.stub :new, stubbed_service do
+            storage = Google::Cloud::Storage.new emulator_host: emulator_host
+            _(storage).must_be_kind_of Google::Cloud::Storage::Project
+            _(storage.project).must_equal ""
+          end
+        end
+      end
+    end
+
+    it "honors provided credentials when using an emulator" do
+      stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
+        max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
+        _(project).must_equal "project-id"
+        _(credentials).must_equal default_credentials
+        _(host).must_equal emulator_host
+        OpenStruct.new project: project
+      }
+
+      # Clear all environment variables
+      ENV.stub :[], nil do
+        Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do
+          Google::Cloud::Storage::Service.stub :new, stubbed_service do
+            storage = Google::Cloud::Storage.new credentials: default_credentials, emulator_host: emulator_host
+            _(storage).must_be_kind_of Google::Cloud::Storage::Project
+            _(storage.project).must_equal "project-id"
+          end
+        end
+      end
+    end
+  end
+
   describe "Storage.configure" do
     let(:found_credentials) { "{}" }
 
@@ -660,6 +757,31 @@ def creds.is_a? target
       end
     end
 
+    it "uses storage config for emulator_host" do
+      stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil,
+        max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) {
+        _(project).must_equal "project-id"
+        _(credentials).must_be :nil?
+        _(host).must_equal "http://localhost:9000"
+        OpenStruct.new project: project
+      }
+
+      # Clear all environment variables
+      ENV.stub :[], nil do
+        # Set new configuration
+        Google::Cloud::Storage.configure do |config|
+          config.project_id = "project-id"
+          config.emulator_host = "http://localhost:9000"
+        end
+
+        Google::Cloud::Storage::Service.stub :new, stubbed_service do
+          storage = Google::Cloud::Storage.new
+          _(storage).must_be_kind_of Google::Cloud::Storage::Project
+          _(storage.project).must_equal "project-id"
+        end
+      end
+    end
+
     it "uses storage config for quota project" do
       stubbed_credentials = ->(keyfile, scope: nil) {
         _(keyfile).must_equal "path/to/keyfile.json"