diff --git a/google-cloud-storage/lib/google-cloud-storage.rb b/google-cloud-storage/lib/google-cloud-storage.rb index 1dba01af5dcc..c755002140ce 100644 --- a/google-cloud-storage/lib/google-cloud-storage.rb +++ b/google-cloud-storage/lib/google-cloud-storage.rb @@ -77,7 +77,8 @@ module Cloud # readonly_storage = gcloud.storage scope: readonly_scope # def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, - max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil + max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, + emulator_host: nil Google::Cloud.storage @project, @keyfile, scope: scope, retries: retries || @retries, timeout: timeout || @timeout, @@ -88,7 +89,8 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time base_interval: base_interval, max_interval: max_interval, multiplier: multiplier, - upload_chunk_size: upload_chunk_size + upload_chunk_size: upload_chunk_size, + emulator_host: emulator_host end ## @@ -142,7 +144,7 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time def self.storage project_id = nil, credentials = nil, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, - upload_chunk_size: nil + upload_chunk_size: nil, emulator_host: nil require "google/cloud/storage" Google::Cloud::Storage.new project_id: project_id, credentials: credentials, @@ -156,7 +158,8 @@ def self.storage project_id = nil, credentials = nil, scope: nil, base_interval: base_interval, max_interval: max_interval, multiplier: multiplier, - upload_chunk_size: upload_chunk_size + upload_chunk_size: upload_chunk_size, + emulator_host: emulator_host end end end @@ -172,6 +175,9 @@ def self.storage project_id = nil, credentials = nil, scope: nil, "STORAGE_KEYFILE", "STORAGE_KEYFILE_JSON" ) end + default_emulator = Google::Cloud::Config.deferred do + ENV["STORAGE_EMULATOR_HOST"] + end config.add_field! :project_id, default_project, match: String, allow_nil: true config.add_alias! :project, :project_id @@ -192,5 +198,6 @@ def self.storage project_id = nil, credentials = nil, scope: nil, config.add_field! :send_timeout, nil, match: Integer config.add_field! :upload_chunk_size, nil, match: Integer config.add_field! :endpoint, nil, match: String, allow_nil: true + config.add_field! :emulator_host, default_emulator, match: String, allow_nil: true config.add_field! :universe_domain, nil, match: String, allow_nil: true end diff --git a/google-cloud-storage/lib/google/cloud/storage.rb b/google-cloud-storage/lib/google/cloud/storage.rb index 8da4ebb5ed7d..5d28347970c4 100644 --- a/google-cloud-storage/lib/google/cloud/storage.rb +++ b/google-cloud-storage/lib/google/cloud/storage.rb @@ -70,6 +70,11 @@ module Storage # @param [Integer] send_timeout How long, in seconds, before receiving response from server times out. Optional. # @param [String] endpoint Override of the endpoint host name. Optional. # If the param is nil, uses the default endpoint. + # @param [String] emulator_host Address of a Storage emulator to connect + # to, including the scheme (e.g. `http://localhost:9000`). Optional. + # Defaults to the `STORAGE_EMULATOR_HOST` environment variable when set. + # When present, the client connects to the emulator and does not require + # credentials (it connects anonymously unless credentials are provided). # @param universe_domain [String] Override of the universe domain. Optional. # If unset or nil, uses the default unvierse domain # @param [Integer] upload_chunk_size The chunk size of storage upload, in bytes. @@ -97,7 +102,8 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, endpoint: nil, project: nil, keyfile: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, - multiplier: nil, upload_chunk_size: nil, universe_domain: nil + multiplier: nil, upload_chunk_size: nil, universe_domain: nil, + emulator_host: nil scope ||= configure.scope retries ||= configure.retries timeout ||= configure.timeout @@ -105,20 +111,27 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, read_timeout ||= configure.read_timeout || timeout send_timeout ||= configure.send_timeout || timeout endpoint ||= configure.endpoint - credentials ||= keyfile || default_credentials(scope: scope) max_elapsed_time ||= configure.max_elapsed_time base_interval ||= configure.base_interval max_interval ||= configure.max_interval multiplier ||= configure.multiplier upload_chunk_size ||= configure.upload_chunk_size universe_domain ||= configure.universe_domain + emulator_host ||= configure.emulator_host - unless credentials.is_a? Google::Auth::Credentials + if emulator_host + endpoint = emulator_host + credentials ||= keyfile + else + credentials ||= keyfile || default_credentials(scope: scope) + end + + if credentials && !credentials.is_a?(Google::Auth::Credentials) credentials = Storage::Credentials.new credentials, scope: scope end project_id = resolve_project_id(project_id || project, credentials) - raise ArgumentError, "project_id is missing" if project_id.empty? + raise ArgumentError, "project_id is missing" if project_id.empty? && !emulator_host Storage::Project.new( Storage::Service.new( @@ -203,6 +216,10 @@ def self.anonymous retries: nil, timeout: nil, open_timeout: nil, # parameter `keyfile` is considered deprecated, but may also be used.) # * `endpoint` - (String) Override of the endpoint host name, or `nil` # to use the default endpoint. + # * `emulator_host` - (String) Address of a Storage emulator to connect to, + # including the scheme (e.g. `http://localhost:9000`), or `nil` to connect + # to the live service. Defaults to the `STORAGE_EMULATOR_HOST` environment + # variable when set. # * `scope` - (String, Array) The OAuth 2.0 scopes controlling # the set of resources and operations that the connection can access. # * `retries` - (Integer) Number of times to retry requests on server diff --git a/google-cloud-storage/test/google/cloud/storage_test.rb b/google-cloud-storage/test/google/cloud/storage_test.rb index 66033eb11448..0c67f09b5ec0 100644 --- a/google-cloud-storage/test/google/cloud/storage_test.rb +++ b/google-cloud-storage/test/google/cloud/storage_test.rb @@ -19,7 +19,7 @@ describe "#storage" do it "calls out to Google::Cloud.storage" do gcloud = Google::Cloud.new - stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) { + stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) { _(project).must_be :nil? _(keyfile).must_be :nil? _(scope).must_be :nil? @@ -44,7 +44,7 @@ it "passes project and keyfile to Google::Cloud.storage" do gcloud = Google::Cloud.new "project-id", "keyfile-path" - stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) { + stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) { _(project).must_equal "project-id" _(keyfile).must_equal "keyfile-path" _(scope).must_be :nil? @@ -69,7 +69,7 @@ it "passes project and keyfile and options to Google::Cloud.storage" do gcloud = Google::Cloud.new "project-id", "keyfile-path" - stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil) { + stubbed_storage = ->(project, keyfile, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, emulator_host: nil) { _(project).must_equal "project-id" _(keyfile).must_equal "keyfile-path" _(scope).must_equal "http://example.com/scope" @@ -387,6 +387,103 @@ def creds.is_a? target end end + describe "Storage.emulator" do + let(:emulator_host) { "http://localhost:9000" } + let(:default_credentials) do + creds = OpenStruct.new empty: true + def creds.is_a? target + target == Google::Auth::Credentials + end + creds + end + + it "uses STORAGE_EMULATOR_HOST environment variable" do + emulator_check = ->(name) { (name == "STORAGE_EMULATOR_HOST") ? emulator_host : nil } + stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + _(project).must_equal "project-id" + _(credentials).must_be :nil? + _(host).must_equal emulator_host + OpenStruct.new project: project + } + + # Clear all environment variables, except STORAGE_EMULATOR_HOST + ENV.stub :[], emulator_check do + # Get project_id from Google Compute Engine + Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do + Google::Cloud::Storage::Service.stub :new, stubbed_service do + storage = Google::Cloud::Storage.new + _(storage).must_be_kind_of Google::Cloud::Storage::Project + _(storage.project).must_equal "project-id" + end + end + end + end + + it "allows emulator_host to be set" do + stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + _(project).must_equal "project-id" + _(credentials).must_be :nil? + _(host).must_equal emulator_host + OpenStruct.new project: project + } + + # Clear all environment variables + ENV.stub :[], nil do + Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do + Google::Cloud::Storage::Service.stub :new, stubbed_service do + storage = Google::Cloud::Storage.new emulator_host: emulator_host + _(storage).must_be_kind_of Google::Cloud::Storage::Project + _(storage.project).must_equal "project-id" + end + end + end + end + + it "does not require a project_id when using an emulator" do + stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + _(project).must_equal "" + _(credentials).must_be :nil? + _(host).must_equal emulator_host + OpenStruct.new project: project + } + + # Clear all environment variables, and provide no project_id from any source + ENV.stub :[], nil do + Google::Cloud.stub :env, OpenStruct.new(project_id: nil) do + Google::Cloud::Storage::Service.stub :new, stubbed_service do + storage = Google::Cloud::Storage.new emulator_host: emulator_host + _(storage).must_be_kind_of Google::Cloud::Storage::Project + _(storage.project).must_equal "" + end + end + end + end + + it "honors provided credentials when using an emulator" do + stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + _(project).must_equal "project-id" + _(credentials).must_equal default_credentials + _(host).must_equal emulator_host + OpenStruct.new project: project + } + + # Clear all environment variables + ENV.stub :[], nil do + Google::Cloud.stub :env, OpenStruct.new(project_id: "project-id") do + Google::Cloud::Storage::Service.stub :new, stubbed_service do + storage = Google::Cloud::Storage.new credentials: default_credentials, emulator_host: emulator_host + _(storage).must_be_kind_of Google::Cloud::Storage::Project + _(storage.project).must_equal "project-id" + end + end + end + end + end + describe "Storage.configure" do let(:found_credentials) { "{}" } @@ -660,6 +757,31 @@ def creds.is_a? target end end + it "uses storage config for emulator_host" do + stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + _(project).must_equal "project-id" + _(credentials).must_be :nil? + _(host).must_equal "http://localhost:9000" + OpenStruct.new project: project + } + + # Clear all environment variables + ENV.stub :[], nil do + # Set new configuration + Google::Cloud::Storage.configure do |config| + config.project_id = "project-id" + config.emulator_host = "http://localhost:9000" + end + + Google::Cloud::Storage::Service.stub :new, stubbed_service do + storage = Google::Cloud::Storage.new + _(storage).must_be_kind_of Google::Cloud::Storage::Project + _(storage.project).must_equal "project-id" + end + end + end + it "uses storage config for quota project" do stubbed_credentials = ->(keyfile, scope: nil) { _(keyfile).must_equal "path/to/keyfile.json"