From 3c23d1ff84e99560740f56a46a586a37f985407e Mon Sep 17 00:00:00 2001 From: Matt Zumwalt Date: Wed, 18 Nov 2015 13:35:17 +0100 Subject: [PATCH 1/2] Refines and tests collection create/update/delete commands --- lib/solr_wrapper.rb | 2 + lib/solr_wrapper/instance.rb | 81 ++++++++++++++-- .../{ => conf}/_rest_managed.json | 0 .../basic_configs/{ => conf}/currency.xml | 0 .../{ => conf}/lang/stopwords_en.txt | 0 .../basic_configs/{ => conf}/protwords.txt | 0 .../basic_configs/{ => conf}/schema.xml | 0 .../basic_configs/{ => conf}/solrconfig.xml | 0 .../basic_configs/{ => conf}/stopwords.txt | 0 .../basic_configs/{ => conf}/synonyms.txt | 0 spec/lib/solr_wrapper/instance_spec.rb | 92 ++++++++++++++++++- 11 files changed, 165 insertions(+), 10 deletions(-) rename spec/fixtures/basic_configs/{ => conf}/_rest_managed.json (100%) rename spec/fixtures/basic_configs/{ => conf}/currency.xml (100%) rename spec/fixtures/basic_configs/{ => conf}/lang/stopwords_en.txt (100%) rename spec/fixtures/basic_configs/{ => conf}/protwords.txt (100%) rename spec/fixtures/basic_configs/{ => conf}/schema.xml (100%) rename spec/fixtures/basic_configs/{ => conf}/solrconfig.xml (100%) rename spec/fixtures/basic_configs/{ => conf}/stopwords.txt (100%) rename spec/fixtures/basic_configs/{ => conf}/synonyms.txt (100%) diff --git a/lib/solr_wrapper.rb b/lib/solr_wrapper.rb index 9d46a5a..60ed2d6 100644 --- a/lib/solr_wrapper.rb +++ b/lib/solr_wrapper.rb @@ -2,6 +2,8 @@ require 'solr_wrapper/instance' module SolrWrapper + class CollectionNotFoundError < RuntimeError ; end + class ZookeeperNotRunning < RuntimeError ; end def self.default_solr_version '5.3.1' end diff --git a/lib/solr_wrapper/instance.rb b/lib/solr_wrapper/instance.rb index 07ef501..e5b6fa1 100644 --- a/lib/solr_wrapper/instance.rb +++ b/lib/solr_wrapper/instance.rb @@ -45,6 +45,7 @@ def wrap(&_block) ## # Start Solr and wait for it to become available + # @return [StringIO] output from executing the command def start extract_and_configure if managed? @@ -59,6 +60,7 @@ def start ## # Stop Solr and wait for it to finish exiting + # @return [StringIO] output from executing the command def stop if managed? && started? @@ -74,12 +76,23 @@ def stop ## # Stop Solr and wait for it to finish exiting + # @return [StringIO] output from executing the command def restart if managed? && started? exec('restart', p: port, c: options[:cloud]) end end + ## + # Stop solr and remove the install directory + # Warning: This will delete the entire instance_dir + # @return [String] path to the instance_dir that was deleted + def destroy + stop + FileUtils.rm_rf instance_dir + instance_dir + end + ## # Check the status of a managed Solr service def status @@ -91,33 +104,79 @@ def status ## # Is Solr running? + # @return [Boolean] whether solr is running def started? !!status end ## - # Create a new collection in solr + # Create a new collection (or core) in solr + # @param [String] name of the collection to create (defaults to a generated hex value) # @param [Hash] options - # @option options [String] :name # @option options [String] :dir - def create(options = {}) - options[:name] ||= SecureRandom.hex - + # @return [String] name of the collection created + def create(name=nil, options = {}) + name ||= SecureRandom.hex create_options = { p: port } - create_options[:c] = options[:name] if options[:name] + create_options[:c] = name create_options[:d] = options[:dir] if options[:dir] exec("create", create_options) - options[:name] + name end ## - # Create a new collection in solr + # Delete a collection (or core) from solr # @param [String] name collection name + # @return [StringIO] output from executing the command def delete(name, _options = {}) exec("delete", c: name, p: port) end + ## + # Create or Update a collection (or core) in solr + # It is not possible to 'update' a core. You have + # to delete it and create again. + # @param [String] name collection name + # @option options [String] :dir + # @return [String] name of the collection + def create_or_update(name, options={}) + delete(name, options) if collection_exists?(name) + create(name, options) + end + + ### + # Check whether a collection (or core) exists in solr + # @param [String] name collection name + def collection_exists?(name) + begin + # Delete the collection if it exists + healthcheck(name) + true + rescue SolrWrapper::CollectionNotFoundError + false + end + end + + ### + # Run solr healthcheck command for a collection (or core) + # @param [String] name collection name + def healthcheck(name, _options = {}) + begin + exec("healthcheck", c: name, z:"#{host}:#{zkport}") + rescue RuntimeError => e + case e.message + when /ERROR: Collection #{name} not found!/ + raise SolrWrapper::CollectionNotFoundError, e.message + when /Could not connect to ZooKeeper/, /port out of range/, /org.apache.zookeeper.ClientCnxn\$SendThread; Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect/ + raise SolrWrapper::ZookeeperNotRunning, "Zookeeper is not running at #{host}:#{zkport}. Are you sure solr is running in cloud mode?" + else + raise e + end + end + end + + ## # Create a new collection, run the block, and then clean up the collection # @param [Hash] options @@ -126,7 +185,7 @@ def delete(name, _options = {}) def with_collection(options = {}) return yield if options.empty? - name = create(options) + name = create(options[:name], options) begin yield name ensure @@ -146,6 +205,10 @@ def port @port ||= options.fetch(:port, random_open_port).to_s end + def zkport + @zkport ||= (port.to_i + 1000).to_s + end + ## # Clean up any files solr_wrapper may have downloaded def clean! diff --git a/spec/fixtures/basic_configs/_rest_managed.json b/spec/fixtures/basic_configs/conf/_rest_managed.json similarity index 100% rename from spec/fixtures/basic_configs/_rest_managed.json rename to spec/fixtures/basic_configs/conf/_rest_managed.json diff --git a/spec/fixtures/basic_configs/currency.xml b/spec/fixtures/basic_configs/conf/currency.xml similarity index 100% rename from spec/fixtures/basic_configs/currency.xml rename to spec/fixtures/basic_configs/conf/currency.xml diff --git a/spec/fixtures/basic_configs/lang/stopwords_en.txt b/spec/fixtures/basic_configs/conf/lang/stopwords_en.txt similarity index 100% rename from spec/fixtures/basic_configs/lang/stopwords_en.txt rename to spec/fixtures/basic_configs/conf/lang/stopwords_en.txt diff --git a/spec/fixtures/basic_configs/protwords.txt b/spec/fixtures/basic_configs/conf/protwords.txt similarity index 100% rename from spec/fixtures/basic_configs/protwords.txt rename to spec/fixtures/basic_configs/conf/protwords.txt diff --git a/spec/fixtures/basic_configs/schema.xml b/spec/fixtures/basic_configs/conf/schema.xml similarity index 100% rename from spec/fixtures/basic_configs/schema.xml rename to spec/fixtures/basic_configs/conf/schema.xml diff --git a/spec/fixtures/basic_configs/solrconfig.xml b/spec/fixtures/basic_configs/conf/solrconfig.xml similarity index 100% rename from spec/fixtures/basic_configs/solrconfig.xml rename to spec/fixtures/basic_configs/conf/solrconfig.xml diff --git a/spec/fixtures/basic_configs/stopwords.txt b/spec/fixtures/basic_configs/conf/stopwords.txt similarity index 100% rename from spec/fixtures/basic_configs/stopwords.txt rename to spec/fixtures/basic_configs/conf/stopwords.txt diff --git a/spec/fixtures/basic_configs/synonyms.txt b/spec/fixtures/basic_configs/conf/synonyms.txt similarity index 100% rename from spec/fixtures/basic_configs/synonyms.txt rename to spec/fixtures/basic_configs/conf/synonyms.txt diff --git a/spec/lib/solr_wrapper/instance_spec.rb b/spec/lib/solr_wrapper/instance_spec.rb index 5300bdc..884db5a 100644 --- a/spec/lib/solr_wrapper/instance_spec.rb +++ b/spec/lib/solr_wrapper/instance_spec.rb @@ -17,6 +17,97 @@ end end end + describe 'destroy' do + subject { solr_instance.destroy } + it 'stops solr and deletes the entire instance_dir' do + expect(solr_instance).to receive(:stop) + expect(FileUtils).to receive(:rm_rf).with(solr_instance.instance_dir) + subject + end + end + describe 'cloud commands' do + let(:collection_name) { 'test_collection' } + let(:existing_collection) { solr_instance.create(collection_name) } + let(:collection_config_dir) { File.join(FIXTURES_DIR, "basic_configs") } + let(:solr_instance) { @solr_instance } + before(:all) do + @solr_instance = SolrWrapper::Instance.new(cloud: true) + @solr_instance.start + end + after(:all) do + @solr_instance.stop + end + describe 'create' do + subject { solr_instance.create(collection_name, dir:collection_config_dir) } + after { solr_instance.delete(collection_name) } + it 'creates a collection' do + expect(solr_instance.collection_exists?(collection_name)).to eq false + expect(subject).to eq collection_name + expect(solr_instance.collection_exists?(collection_name)).to eq true + end + end + describe 'delete' do + subject { solr_instance.delete(existing_collection) } + it 'deletes a collection' do + expect(solr_instance.collection_exists?(existing_collection)).to eq true + subject + expect(solr_instance.collection_exists?(existing_collection)).to eq false + end + end + describe 'create_or_update' do + subject { solr_instance.create_or_update(collection_name, dir:collection_config_dir) } + context 'when the collection does not exist' do + before do + expect(solr_instance).to receive(:collection_exists?).and_return(false) + end + it 'creates the collection' do + expect(solr_instance).to_not receive(:delete) + expect(solr_instance).to receive(:create).with(collection_name, dir:collection_config_dir) + subject + end + end + context 'when the collection already exists' do + before do + expect(solr_instance).to receive(:collection_exists?).and_return(true) + end + it 'delete the collection and then creates it again' do + expect(solr_instance).to receive(:delete).with(collection_name, dir:collection_config_dir) + expect(solr_instance).to receive(:create).with(collection_name, dir:collection_config_dir) + subject + end + end + end + describe 'healthcheck' do + context 'when the collection does not exist' do + subject { solr_instance.healthcheck('nonexistent') } + it 'raises an error' do + expect { subject }.to raise_error(SolrWrapper::CollectionNotFoundError) + end + end + context 'when the collection exists' do + subject { solr_instance.healthcheck(existing_collection) } + after { solr_instance.delete(existing_collection) } + it 'returns info about the collection' do + expect(subject).to be_instance_of StringIO + json = JSON.parse(subject.read) + expect(json['collection']).to eq existing_collection + expect(json['status']).to eq 'healthy' + expect(json['numDocs']).to eq 0 + end + end + context 'when zookeeper is not running' do + let(:wrapper_error_message) { "Zookeeper is not running at #{solr_instance.host}:#{solr_instance.zkport}. Are you sure solr is running in cloud mode?" } + it 'raises an appropriate error' do + expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "ERROR: java.lang.IllegalArgumentException: port out of range:65831") + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "org.apache.zookeeper.ClientCnxn$SendThread; Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect") + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "ERROR: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:58499 within 10000 ms") + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + end + end + end + end describe 'exec' do let(:cmd) { 'start' } let(:options) { { p: '4098', help: true } } @@ -29,7 +120,6 @@ result_io = solr_instance.send(:exec, 'start', p: '4098', help: true) expect(result_io.read).to include('Usage: solr start') end - describe 'when something goes wrong' do let(:cmd) { 'healthcheck' } let(:options) { { z: 'localhost:5098' } } From 26bce2ea3d9545065628bf9a0d3e2385afa31892 Mon Sep 17 00:00:00 2001 From: Matt Zumwalt Date: Wed, 18 Nov 2015 14:45:07 +0100 Subject: [PATCH 2/2] adds a reload_collection method --- lib/solr_wrapper.rb | 3 ++- lib/solr_wrapper/instance.rb | 21 ++++++++++++++++++- spec/lib/solr_wrapper/instance_spec.rb | 28 +++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/solr_wrapper.rb b/lib/solr_wrapper.rb index 60ed2d6..649027c 100644 --- a/lib/solr_wrapper.rb +++ b/lib/solr_wrapper.rb @@ -3,7 +3,8 @@ module SolrWrapper class CollectionNotFoundError < RuntimeError ; end - class ZookeeperNotRunning < RuntimeError ; end + class NotInCloudModeError < RuntimeError ; end + class ZookeeperNotRunningError < RuntimeError ; end def self.default_solr_version '5.3.1' end diff --git a/lib/solr_wrapper/instance.rb b/lib/solr_wrapper/instance.rb index e5b6fa1..7f2d62e 100644 --- a/lib/solr_wrapper/instance.rb +++ b/lib/solr_wrapper/instance.rb @@ -145,6 +145,25 @@ def create_or_update(name, options={}) create(name, options) end + ## + # Tell solr to reload a collection and its configuration + # @param [String] name of the collection + def reload_collection(name) + begin + open url + "admin/collections?action=RELOAD&wt=json&name=#{name}" + rescue OpenURI::HTTPError => e + response_body = e.io.read + case response_body + when /Solr instance is not running in SolrCloud mode./ + raise SolrWrapper::NotInCloudModeError, response_body + when /Could not find collection/ + raise SolrWrapper::CollectionNotFoundError, response_body + else + response_body + end + end + end + ### # Check whether a collection (or core) exists in solr # @param [String] name collection name @@ -169,7 +188,7 @@ def healthcheck(name, _options = {}) when /ERROR: Collection #{name} not found!/ raise SolrWrapper::CollectionNotFoundError, e.message when /Could not connect to ZooKeeper/, /port out of range/, /org.apache.zookeeper.ClientCnxn\$SendThread; Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect/ - raise SolrWrapper::ZookeeperNotRunning, "Zookeeper is not running at #{host}:#{zkport}. Are you sure solr is running in cloud mode?" + raise SolrWrapper::ZookeeperNotRunningError, "Zookeeper is not running at #{host}:#{zkport}. Are you sure solr is running in cloud mode?" else raise e end diff --git a/spec/lib/solr_wrapper/instance_spec.rb b/spec/lib/solr_wrapper/instance_spec.rb index 884db5a..9f9b0c5 100644 --- a/spec/lib/solr_wrapper/instance_spec.rb +++ b/spec/lib/solr_wrapper/instance_spec.rb @@ -17,6 +17,28 @@ end end end + describe 'reload_collection' do + let(:collection_name) { 'test_collection' } + let(:not_in_cloud_mode_response) { '{"responseHeader":{"status":400,"QTime":2},"error":{"msg":"Solr instance is not running in SolrCloud mode.","code":400}}' } + let(:collection_not_found_response) { '{"responseHeader"=>{"status"=>400, "QTime"=>42}, "Operation reload caused exception:"=>"org.apache.solr.common.SolrException:org.apache.solr.common.SolrException: Could not find collection : test_collection", "exception"=>{"msg"=>"Could not find collection : test_collection", "rspCode"=>400}, "error"=>{"msg"=>"Could not find collection : test_collection", "code"=>400}}' } + subject { solr_instance.reload_collection(collection_name) } + it 'uses the Collections (REST) API to reload the collection' do + expect(solr_instance).to receive(:open).with(solr_instance.url+"admin/collections?action=RELOAD&wt=json&name=#{collection_name}") + subject + end + it 'when solr is not running raises the Errno::ECONNREFUSED error' do + expect(solr_instance).to receive(:open).and_raise(Errno::ECONNREFUSED) + expect { subject }.to raise_error(Errno::ECONNREFUSED) + end + it 'when solr is not in cloud mode raises a NotInCloudModeError' do + expect(solr_instance).to receive(:open).and_raise(OpenURI::HTTPError.new('message',StringIO.new(not_in_cloud_mode_response))) + expect { subject }.to raise_error(SolrWrapper::NotInCloudModeError) + end + it 'when the collection does not exist raises a CollectionNotFoundError' do + expect(solr_instance).to receive(:open).and_raise(OpenURI::HTTPError.new('message',StringIO.new(collection_not_found_response))) + expect { subject }.to raise_error(SolrWrapper::CollectionNotFoundError) + end + end describe 'destroy' do subject { solr_instance.destroy } it 'stops solr and deletes the entire instance_dir' do @@ -99,11 +121,11 @@ let(:wrapper_error_message) { "Zookeeper is not running at #{solr_instance.host}:#{solr_instance.zkport}. Are you sure solr is running in cloud mode?" } it 'raises an appropriate error' do expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "ERROR: java.lang.IllegalArgumentException: port out of range:65831") - expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunningError, wrapper_error_message) expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "org.apache.zookeeper.ClientCnxn$SendThread; Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect") - expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunningError, wrapper_error_message) expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "ERROR: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:58499 within 10000 ms") - expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunningError, wrapper_error_message) end end end