From 19d0d083419aaa55eff0e8eae569434c61b91a2d Mon Sep 17 00:00:00 2001 From: seancookr Date: Wed, 31 Aug 2016 09:52:17 -0700 Subject: [PATCH 1/7] WIP --- lib/dolly/replicator.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lib/dolly/replicator.rb diff --git a/lib/dolly/replicator.rb b/lib/dolly/replicator.rb new file mode 100644 index 0000000..0777c65 --- /dev/null +++ b/lib/dolly/replicator.rb @@ -0,0 +1,7 @@ +module Dolly + class Replicator + def initialize source, target + + end + end +end From b41c2218a19d21fd6b93cc63e235bfafa8629bbc Mon Sep 17 00:00:00 2001 From: seancookr Date: Fri, 3 Nov 2017 19:56:04 -0700 Subject: [PATCH 2/7] WIP --- lib/dolly/replicator.rb | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/dolly/replicator.rb b/lib/dolly/replicator.rb index 0777c65..2963967 100644 --- a/lib/dolly/replicator.rb +++ b/lib/dolly/replicator.rb @@ -1,7 +1,26 @@ module Dolly - class Replicator - def initialize source, target + class SimpleReplicator + include Dolly::Connection + attr_reader :source_db, :target_db + + def initialize source_db, target_db + @source_db = source_db + @target_db = target_db + end + + def replicate! + database.request :post, '/_replicate', {body: request_body} end + + private + + def request_body + { + source: source_db, + target_db: target_db + }.to_json + end + end end From bfa4d8367d272a67572692f796e07aa593163998 Mon Sep 17 00:00:00 2001 From: seancookr Date: Mon, 6 Nov 2017 11:42:06 -0800 Subject: [PATCH 3/7] WIP --- lib/dolly/replicator/database.rb | 15 +++++++++++++++ lib/dolly/{replicator.rb => simple_replicator.rb} | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 lib/dolly/replicator/database.rb rename lib/dolly/{replicator.rb => simple_replicator.rb} (91%) diff --git a/lib/dolly/replicator/database.rb b/lib/dolly/replicator/database.rb new file mode 100644 index 0000000..c460516 --- /dev/null +++ b/lib/dolly/replicator/database.rb @@ -0,0 +1,15 @@ +require "dolly/request" +require "dolly/db_config" + +module Dolly + module Replicator + module Database + + def database + options = {'name' => '_replicator'} + replicator_env = env.merge options + Request.new(replicator_env) + end + end + end +end diff --git a/lib/dolly/replicator.rb b/lib/dolly/simple_replicator.rb similarity index 91% rename from lib/dolly/replicator.rb rename to lib/dolly/simple_replicator.rb index 2963967..0ecc7a2 100644 --- a/lib/dolly/replicator.rb +++ b/lib/dolly/simple_replicator.rb @@ -1,6 +1,6 @@ module Dolly class SimpleReplicator - include Dolly::Connection + include Dolly::Replicator::Database attr_reader :source_db, :target_db From 478eecf6fa202ca4bcdab94e35de095e001ad25f Mon Sep 17 00:00:00 2001 From: seancookr Date: Mon, 6 Nov 2017 11:53:13 -0800 Subject: [PATCH 4/7] add replicate! to the api of Dolly::Request --- lib/dolly/request.rb | 4 ++++ lib/dolly/simple_replicator.rb | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/dolly/request.rb b/lib/dolly/request.rb index 93ea44a..969a948 100644 --- a/lib/dolly/request.rb +++ b/lib/dolly/request.rb @@ -80,6 +80,10 @@ def request method, resource, data = nil end end + def replicate! target_db, opts={} + Dolly::SimpleReplicator.new(self, target_db, opts).replicate! + end + private def tools path, opts = nil data = {} diff --git a/lib/dolly/simple_replicator.rb b/lib/dolly/simple_replicator.rb index 0ecc7a2..8a3b462 100644 --- a/lib/dolly/simple_replicator.rb +++ b/lib/dolly/simple_replicator.rb @@ -4,7 +4,8 @@ class SimpleReplicator attr_reader :source_db, :target_db - def initialize source_db, target_db + def initialize source_db, target_db, opts={} + #@opts = opts #TODO add options for difference replications @source_db = source_db @target_db = target_db end From 22621abfbf5ca70cd9b178af679e53424149b188 Mon Sep 17 00:00:00 2001 From: seancookr Date: Fri, 26 Jan 2018 11:15:29 -0800 Subject: [PATCH 5/7] abstract a base class so that I can add replicator to request --- lib/dolly/replicator/database.rb | 15 ---- lib/dolly/request.rb | 118 +------------------------------ lib/dolly/simple_replicator.rb | 4 +- 3 files changed, 5 insertions(+), 132 deletions(-) delete mode 100644 lib/dolly/replicator/database.rb diff --git a/lib/dolly/replicator/database.rb b/lib/dolly/replicator/database.rb deleted file mode 100644 index c460516..0000000 --- a/lib/dolly/replicator/database.rb +++ /dev/null @@ -1,15 +0,0 @@ -require "dolly/request" -require "dolly/db_config" - -module Dolly - module Replicator - module Database - - def database - options = {'name' => '_replicator'} - replicator_env = env.merge options - Request.new(replicator_env) - end - end - end -end diff --git a/lib/dolly/request.rb b/lib/dolly/request.rb index 969a948..857591a 100644 --- a/lib/dolly/request.rb +++ b/lib/dolly/request.rb @@ -1,123 +1,9 @@ -require "httparty" -require "dolly/bulk_document" +require "dolly/simple_replicator" module Dolly - - class Request - include HTTParty - REQUIRED_KEYS = %w/host port name/.freeze - - attr_accessor :database_name, :host, :port, :bulk_document - - def initialize options = {} - REQUIRED_KEYS.each do |key| - raise Dolly::MissingRequestConfigSettings.new(key) unless options[key] - end - - @host = options["host"] - @port = options["port"] - @database_name = options["name"] - @username = options["username"] - @password = options["password"] - @protocol = options["protocol"] - - @bulk_document = Dolly::BulkDocument.new [] - self.class.base_uri "#{protocol}://#{host}:#{port}" - end - - def get resource, data = nil - q = {query: values_to_json(data)} if data - request :get, full_path(resource), q - end - - def stats - request :get, "/#{database_name}" - end - - def put resource, data - request :put, full_path(resource), {body: data} - end - - def post resource, data - request :post, full_path(resource), {body: data} - end - - def delete resource - request :delete, full_path(resource), {} - end - - def attach resource, attachment_name, data, headers = {} - data = StringIO.new(data) if data.is_a?(String) - request :put, attachment_path(resource, attachment_name), {body: data, headers: headers} - end - - def protocol - @protocol ||= 'http' - end - - def uuids opts = {} - tools("_uuids", opts)["uuids"] - end - - def all_docs data = {} - data = values_to_json data.merge( include_docs: true ) - request :get, full_path('_all_docs'), {query: data} - end - - def request method, resource, data = nil - data ||= {} - data.merge!(basic_auth: auth_info) if auth_info.present? - headers = { 'Content-Type' => 'application/json' } - headers.merge! data[:headers] if data[:headers] - response = self.class.send method, resource, data.merge(headers: headers) - log_request(resource, response.code) if Dolly.log_requests? - if response.code == 404 - raise Dolly::ResourceNotFound - elsif (400..600).include? response.code - raise Dolly::ServerError.new( response ) - else - response - end - end - + class Request < Dolly::Requests::Base def replicate! target_db, opts={} Dolly::SimpleReplicator.new(self, target_db, opts).replicate! end - - private - def tools path, opts = nil - data = {} - q = "?#{CGI.unescape(opts.to_query)}" unless opts.blank? - data.merge!(basic_auth: auth_info) if auth_info.present? - JSON::parse self.class.get("/#{path}#{q}", data) - end - - def auth_info - return nil unless @username.present? - {username: @username, password: @password} - end - - def values_to_json hash - hash.reduce({}){|h, v| h[v.first] = v.last.to_json; h} - end - - def full_path resource - "/#{database_name}/#{resource}" - end - - def attachment_path resource, attachment_name - "#{full_path(resource)}/#{attachment_name}" - end - - def log_request resource, response_code - log_value = ->(resource, response_code) { "Query: #{resource}, Response Code: #{response_code}" } - case response_code - when 200..399 - Dolly.logger.info log_value[resource, response_code] - when 400..600 - Dolly.logger.warn log_value[resource, response_code] - end - end end - end diff --git a/lib/dolly/simple_replicator.rb b/lib/dolly/simple_replicator.rb index 8a3b462..1249d8e 100644 --- a/lib/dolly/simple_replicator.rb +++ b/lib/dolly/simple_replicator.rb @@ -1,6 +1,8 @@ +require 'dolly/replication/database' + module Dolly class SimpleReplicator - include Dolly::Replicator::Database + include Dolly::Replication::Database attr_reader :source_db, :target_db From 261d0f1ef7ac32e8bc33595eb94ff0344d60a810 Mon Sep 17 00:00:00 2001 From: seancookr Date: Fri, 26 Jan 2018 11:15:36 -0800 Subject: [PATCH 6/7] abstract a base class so that I can add replicator to request --- lib/dolly/replication.rb | 7 ++ lib/dolly/replication/database.rb | 15 ++++ lib/dolly/requests/base.rb | 121 ++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 lib/dolly/replication.rb create mode 100644 lib/dolly/replication/database.rb create mode 100644 lib/dolly/requests/base.rb diff --git a/lib/dolly/replication.rb b/lib/dolly/replication.rb new file mode 100644 index 0000000..a3960ec --- /dev/null +++ b/lib/dolly/replication.rb @@ -0,0 +1,7 @@ +require 'dolly/replication/database' + +module Dolly + module Replication + include Dolly::Replication::Database + end +end diff --git a/lib/dolly/replication/database.rb b/lib/dolly/replication/database.rb new file mode 100644 index 0000000..5fdc2d8 --- /dev/null +++ b/lib/dolly/replication/database.rb @@ -0,0 +1,15 @@ +require "dolly/requests/base" +require "dolly/db_config" + +module Dolly + module Replication + module Database + + def database + options = {'name' => '_replicator'} + replicator_env = env.merge options + Request::Base.new(replicator_env) + end + end + end +end diff --git a/lib/dolly/requests/base.rb b/lib/dolly/requests/base.rb new file mode 100644 index 0000000..9a2cf60 --- /dev/null +++ b/lib/dolly/requests/base.rb @@ -0,0 +1,121 @@ +require "httparty" +require "dolly/bulk_document" + +module Dolly + + module Requests + class Base + include HTTParty + REQUIRED_KEYS = %w/host port name/.freeze + + attr_accessor :database_name, :host, :port, :bulk_document + + def initialize options = {} + REQUIRED_KEYS.each do |key| + raise Dolly::MissingRequestConfigSettings.new(key) unless options[key] + end + + @host = options["host"] + @port = options["port"] + @database_name = options["name"] + @username = options["username"] + @password = options["password"] + @protocol = options["protocol"] + + @bulk_document = Dolly::BulkDocument.new [] + self.class.base_uri "#{protocol}://#{host}:#{port}" + end + + def get resource, data = nil + q = {query: values_to_json(data)} if data + request :get, full_path(resource), q + end + + def stats + request :get, "/#{database_name}" + end + + def put resource, data + request :put, full_path(resource), {body: data} + end + + def post resource, data + request :post, full_path(resource), {body: data} + end + + def delete resource + request :delete, full_path(resource), {} + end + + def attach resource, attachment_name, data, headers = {} + data = StringIO.new(data) if data.is_a?(String) + request :put, attachment_path(resource, attachment_name), {body: data, headers: headers} + end + + def protocol + @protocol ||= 'http' + end + + def uuids opts = {} + tools("_uuids", opts)["uuids"] + end + + def all_docs data = {} + data = values_to_json data.merge( include_docs: true ) + request :get, full_path('_all_docs'), {query: data} + end + + def request method, resource, data = nil + data ||= {} + data.merge!(basic_auth: auth_info) if auth_info.present? + headers = { 'Content-Type' => 'application/json' } + headers.merge! data[:headers] if data[:headers] + response = self.class.send method, resource, data.merge(headers: headers) + log_request(resource, response.code) if Dolly.log_requests? + if response.code == 404 + raise Dolly::ResourceNotFound + elsif (400..600).include? response.code + raise Dolly::ServerError.new( response ) + else + response + end + end + + private + def tools path, opts = nil + data = {} + q = "?#{CGI.unescape(opts.to_query)}" unless opts.blank? + data.merge!(basic_auth: auth_info) if auth_info.present? + JSON::parse self.class.get("/#{path}#{q}", data) + end + + def auth_info + return nil unless @username.present? + {username: @username, password: @password} + end + + def values_to_json hash + hash.reduce({}){|h, v| h[v.first] = v.last.to_json; h} + end + + def full_path resource + "/#{database_name}/#{resource}" + end + + def attachment_path resource, attachment_name + "#{full_path(resource)}/#{attachment_name}" + end + + def log_request resource, response_code + log_value = ->(resource, response_code) { "Query: #{resource}, Response Code: #{response_code}" } + case response_code + when 200..399 + Dolly.logger.info log_value[resource, response_code] + when 400..600 + Dolly.logger.warn log_value[resource, response_code] + end + end + end + end + +end From 461c0cfc8ea550ff774ec631f706451945365fef Mon Sep 17 00:00:00 2001 From: seancookr Date: Fri, 26 Jan 2018 11:59:18 -0800 Subject: [PATCH 7/7] add replicate! method with test --- lib/dolly/replication/database.rb | 3 ++- lib/dolly/simple_replicator.rb | 6 +++--- test/request_test.rb | 30 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/dolly/replication/database.rb b/lib/dolly/replication/database.rb index 5fdc2d8..9d3f327 100644 --- a/lib/dolly/replication/database.rb +++ b/lib/dolly/replication/database.rb @@ -4,11 +4,12 @@ module Dolly module Replication module Database + include Dolly::DbConfig def database options = {'name' => '_replicator'} replicator_env = env.merge options - Request::Base.new(replicator_env) + Requests::Base.new(replicator_env) end end end diff --git a/lib/dolly/simple_replicator.rb b/lib/dolly/simple_replicator.rb index 1249d8e..2c06509 100644 --- a/lib/dolly/simple_replicator.rb +++ b/lib/dolly/simple_replicator.rb @@ -13,15 +13,15 @@ def initialize source_db, target_db, opts={} end def replicate! - database.request :post, '/_replicate', {body: request_body} + database.request :post, '/_replicate', body: request_body end private def request_body { - source: source_db, - target_db: target_db + source: source_db.database_name, + target: target_db.database_name }.to_json end diff --git a/test/request_test.rb b/test/request_test.rb index beddc4c..9a72827 100644 --- a/test/request_test.rb +++ b/test/request_test.rb @@ -22,4 +22,34 @@ def setup Dolly::Request.new @default_settings.dup.delete('port') end end + + class ReplicateTest < RequestTest + test 'posts a replication request to the replicator database' do + db = Dolly::Request.new @default_settings + other_db = Dolly::Request.new @default_settings.merge({'name' => 'database_2'}) + resp = { + "ok" => true, + "session_id" => "8fa6f7bfdad58a9aa229d6c482a06012", + "source_last_seq" => 5, + "replication_id_version" => 3, + "history" => [ + { + "session_id" => "8fa6f7bfdad58a9aa229d6c482a06012", + "start_time" => Date.today.to_s, + "end_time" => Date.today.to_s, + "start_last_seq" => 0, + "end_last_seq" => 5, + "recorded_seq" => 5, + "missing_checked" => 4, + "missing_found" => 0, + "docs_read" => 0, + "docs_written" => 0, + "doc_write_failures" => 0 + } + ] + } + FakeWeb.register_uri :post, "http://localhost:5984/_replicate", body: resp.to_json + assert db.replicate!(other_db) + end + end end