From 681e3721ccf69458366fe8a11d479877090e9e80 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Tue, 10 May 2022 16:04:42 +0100 Subject: [PATCH 01/21] Update Gemfile --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index a50a09c..65c57de 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,6 @@ gem 'elasticsearch' gem 'rest-client' gem 'dotenv' -gem 'openactive', :git => 'git://github.com/theodi/openactive.rb' +gem 'openactive', :git => 'https://github.com/theodi/openactive.rb' From 66af3be6bfa1d846c2903550606c0c6007c81f80 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Tue, 10 May 2022 16:05:17 +0100 Subject: [PATCH 02/21] Update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 11233ef..0281e97 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,5 +1,5 @@ GIT - remote: git://github.com/theodi/openactive.rb + remote: https://github.com/theodi/openactive.rb revision: 17fb014a4aca6e0cf003e9da17fa8e4ef9579cac specs: openactive (0.0.1) From 9f11862560aeee531ae6d53aad40bbb06e9462b9 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Tue, 10 May 2022 16:49:34 +0100 Subject: [PATCH 03/21] Update README.md --- README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ae0d333..bbd1a10 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,17 @@ feed. So edit the following section in `config/datasets.json` so that the `index ## Start ElasticSearch -We then need to startup ElasticSearch so we can configure some indexes to hold the data. In a separate terminal -window, from the project directory you can start ElasticSearch by running the following command: +By default, ElasticSearch starts with SSL security restrictions configured in `config/elasticsearch.yml`. For this example, these can be set to `false` rather than `true` as below: + +``` +# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents +xpack.security.http.ssl: + enabled: false + keystore.path: certs/http.p12 +``` + +We then need to startup ElasticSearch so we can configure some indexes to hold the data. +**Open a separate terminal window**, cd to the project directory and start ElasticSearch by running the following command: ``` rake es:start @@ -83,8 +92,7 @@ rake es:start This just runs `./server/bin/elasticsearch` so you can run that directly if you prefer. -You can test its running by visiting `http://localhost:9200/`. You should see a JSON response from your local -ElasticSearch server. +Test it is running by visiting `http://localhost:9200/`. You may have to log in with the user 'elastic' and the password shown in the terminal log. You should see a JSON response from your local ElasticSearch server. You can Ctrl-C to shutdown the server at any time. But it needs to be running for the following steps. @@ -104,7 +112,7 @@ See the [elasticsearch-ruby configuration](http://www.rubydoc.info/gems/elastics ## Create Elastic Search Indexes -Run the following to create the ElasticSearch indexes: +In the original terminal, run the following to create the ElasticSearch indexes: ``` rake es:indexes From 20f5688d68387ad97dfe7d6c90b1f678b8fe75bd Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 15:13:15 +0100 Subject: [PATCH 04/21] Update index-template.json --- config/index-template.json | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/config/index-template.json b/config/index-template.json index d75c920..7be5e02 100644 --- a/config/index-template.json +++ b/config/index-template.json @@ -1,9 +1,12 @@ { - "template": "oa-*", - "mappings": { - "opp": { - "properties": { - } - } + "index_patterns": [ + "oa-*" + ], + "mappings": { + "properties": { + "name" : { + "type" : "text" + } } -} + } + } From f34626ff703ed4262fee16a3e84fc02fb0b6f9d7 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 15:47:25 +0100 Subject: [PATCH 05/21] Update harvest.rb --- bin/harvest.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/harvest.rb b/bin/harvest.rb index b28b22c..7bf7146 100644 --- a/bin/harvest.rb +++ b/bin/harvest.rb @@ -37,7 +37,7 @@ #add each item to the update, using the appropriate index #we use the unique id for the item in the feed as the document id #TODO: we should be checking the state of the item and processing deletes here - body << { index: { _index: "#{INDEX_PREFIX}-#{id}", _type: "opp", _id: item["id"] } } + body << { index: { _index: "#{INDEX_PREFIX}-#{id}", _id: item["id"] } } #here we just add the data about the item that was included in the feed #in a production application you will probably want to process this data to ensure that From 332f4f876e5a972447d113b8894c98e8600473a6 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 15:53:00 +0100 Subject: [PATCH 06/21] Update README.md --- README.md | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index bbd1a10..5fd305c 100644 --- a/README.md +++ b/README.md @@ -61,13 +61,13 @@ rake prepare:config This downloads the current list of published datasets and stores it in `config/datasets.json`. By default indexing is disabled for all datasets, you will need to edit `config/datasets.json` to switch on -whichever datasets you want to try indexing. We suggest trying using the Leisure World Colchester data as its a small +whichever datasets you want to try indexing. We suggest trying using the British Triathlon data as its a relatively small feed. So edit the following section in `config/datasets.json` so that the `index` key is `true` rather than `false`. ``` - "leisureworldcolchester.github.io": { - "title": "Leisure World Colchester Sessions", - "data_url": "https://lw-colchester-openactive.herokuapp.com", + "britishtriathlon.github.io": { + "title": "British Triathlon Events", + "data_url": "https://api.britishtriathlon.org/openactive/v1/events", "index": true } ``` @@ -200,33 +200,16 @@ http://localhost:9200/_stats ``` Then ElasticSearch will dump the current state of all indexes, including how many documents are in each. You can also -ask for index specific statistics. So, assuming you have indexes Leisure World Colchester, then if you visit this URL: +ask for index specific statistics. So, assuming you have indexed British Triathlon, then if you visit this URL: ``` -http://localhost:9200/oa-leisureworldcolchester.github.io/_stats +http://localhost:9200/oa-britishtriathlon.github.io/_stats ``` Then you should get stats for just that index. You should see something like this: ``` -{ - "_shards": { - ... - }, - "_all": { - ... - }, - "indices": { - "oa-leisureworldcolchester.github.io": { - "primaries": { - "docs": { - "count": 156, - "deleted": 0 - }, - ... - } - } -} +{"_shards":{"total":2,"successful":1,"failed":0},"_all":{"primaries":{"docs":{"count":1825,"deleted":0}... ``` Which tells you that 156 records have been indexed. @@ -234,7 +217,7 @@ Which tells you that 156 records have been indexed. You can also visit the `_search` endpoint to see all the docs: ``` -http://localhost:9200/oa-leisureworldcolchester.github.io/_search +http://localhost:9200/oa-britishtriathlon.github.io/_search ``` We suggest reading [the ElasticSearch documentation on their search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/_exploring_your_data.html) From 129ebf291524434e051973559d36a1d52f865771 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 15:53:52 +0100 Subject: [PATCH 07/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5fd305c..7966550 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ Then you should get stats for just that index. You should see something like thi {"_shards":{"total":2,"successful":1,"failed":0},"_all":{"primaries":{"docs":{"count":1825,"deleted":0}... ``` -Which tells you that 156 records have been indexed. +Which tells you that 1825 records have been indexed. You can also visit the `_search` endpoint to see all the docs: From b48bade5610c8672188962568c22ddd123530afe Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 15:56:49 +0100 Subject: [PATCH 08/21] Update Gemfile --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 65c57de..2ea1636 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,5 @@ source 'https://rubygems.org' -ruby "2.4.0" +ruby "2.6.10" gem 'json' gem 'elasticsearch' From 15b06560fb289bfdfa976b87265ddb6e1da465dd Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 15:57:45 +0100 Subject: [PATCH 09/21] Update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0281e97..cbc659d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ DEPENDENCIES rest-client RUBY VERSION - ruby 2.4.0p0 + ruby 2.6.10p210 BUNDLED WITH 1.14.6 From 11ca48a4f5aa7f6e2cb0745254b6e8421a78e506 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 11 May 2022 16:01:48 +0100 Subject: [PATCH 10/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7966550..f61ff58 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ setup has been successfully used to do some simple analysing and reporting on pu You will need to install: -* Ruby -- this was built and tested using Ruby 2.4.0 but should work on later rubies) +* Ruby -- this was built with Ruby 2.4.0 and updated and tested recently with Ruby 2.6.10 * Bundler -- to install other ruby dependencies * Java 1.8+ -- ElasticSearch is a java application From ce64992bb107c0d3450fa9a4dea1e129d7c3b9f4 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Thu, 12 May 2022 11:33:29 +0100 Subject: [PATCH 11/21] Update README.md --- README.md | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f61ff58..fb6757e 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,24 @@ setup has been successfully used to do some simple analysing and reporting on pu You will need to install: -* Ruby -- this was built with Ruby 2.4.0 and updated and tested recently with Ruby 2.6.10 +* Ruby -- this was built with Ruby 2.4.0 and updated recently with the nearest stable version Ruby 2.6.10 * Bundler -- to install other ruby dependencies * Java 1.8+ -- ElasticSearch is a java application +You can use a software verison management tool like chruby to install different versions of ruby and switch between them as needed. If chruby is installed, you can run the following to switch ruby versions and confirm you are using 2.6.10: + +``` +chruby 2.6.10 +ruby -v +``` + ## Grab this code -Clone this repo to your machine, then install the ruby dependencies: +To clone this repo to your machine and install the ruby dependencies, run the following lines in a terminal: ``` +git clone https://github.com/howaskew/openactive-es-example.git +cd openactive-es-example bundle install ``` @@ -31,8 +40,7 @@ bundle install Go to [the Elastic Search download page](https://www.elastic.co/downloads/elasticsearch) and download the zip file of the latest release. -You will need to unzip the file into the `server` sub-directory of this project. You should ignore the -main sub-directory in the zip file, you just need to extract the main project folders. +Unzip this file and copy the contents into the `server` sub-directory of the openactive-es-example directory. You should end up with a `server` directory that looks something like: @@ -41,7 +49,8 @@ server/bin server/config server/lib server/modules -...etc +server/logs +server/plugins ``` ## Update the list of datasets @@ -74,16 +83,16 @@ feed. So edit the following section in `config/datasets.json` so that the `index ## Start ElasticSearch -By default, ElasticSearch starts with SSL security restrictions configured in `config/elasticsearch.yml`. For this example, these can be set to `false` rather than `true` as below: +By default, ElasticSearch starts with security restrictions configured in `config/elasticsearch.yml`. For production applications, disabling security is not recommended but for this example, these can be set to `false` rather than `true` as below: ``` -# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents -xpack.security.http.ssl: - enabled: false - keystore.path: certs/http.p12 +# Enable security features +xpack.security.enabled: false ``` -We then need to startup ElasticSearch so we can configure some indexes to hold the data. +We then need to startup ElasticSearch so we can configure some indexes to hold the data. + + **Open a separate terminal window**, cd to the project directory and start ElasticSearch by running the following command: ``` @@ -92,11 +101,14 @@ rake es:start This just runs `./server/bin/elasticsearch` so you can run that directly if you prefer. -Test it is running by visiting `http://localhost:9200/`. You may have to log in with the user 'elastic' and the password shown in the terminal log. You should see a JSON response from your local ElasticSearch server. +Make a note of the password for the 'elastic' user. + + +Test it is running by visiting `http://localhost:9200/`. Log in with the user 'elastic' and the password shown in the terminal log. You should see a JSON response from your local ElasticSearch server. You can Ctrl-C to shutdown the server at any time. But it needs to be running for the following steps. -ElasticSearch is configured via its API so you need to have an instance available. +ElasticSearch is configured via its API so you need to have a running nstance available. ### Aside: Using a different ElasticSearch Server @@ -112,7 +124,7 @@ See the [elasticsearch-ruby configuration](http://www.rubydoc.info/gems/elastics ## Create Elastic Search Indexes -In the original terminal, run the following to create the ElasticSearch indexes: +**In the original terminal window**, run the following to create the ElasticSearch indexes: ``` rake es:indexes From b926448eb864124c0627cc762581666fcc0916b0 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Thu, 12 May 2022 15:30:03 +0100 Subject: [PATCH 12/21] Update README.md --- README.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fb6757e..47520b8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ This repository contains a simple demonstration of harvesting and indexing [opportunity data](http://status.openactive.io/), published as part of the [OpenActive](https://openactive.io) initiative. The example uses a set of simple Ruby scripts to drive harvesting of live data feeds and indexes the data in -[ElasticSearch](https://www.elastic.co/) which is an open source search engine. +[ElasticSearch](https://www.elastic.co/) and Kibana, open source search tools. The code in this project is published under an open licence and you are free to adapt and reuse it as you see fit. @@ -35,7 +35,7 @@ cd openactive-es-example bundle install ``` -## Download and unzip Elastic Search +## Download and unzip Elastic Search and Kibana Go to [the Elastic Search download page](https://www.elastic.co/downloads/elasticsearch) and download the zip file of the latest release. @@ -53,6 +53,11 @@ server/logs server/plugins ``` +Go to the kibana download page and download the latest version for your platform. +Unzip this and move the contents into the kibana subfolder. + + + ## Update the list of datasets The ruby scripts that manage the indexes and harvest the data are driven off a list of published datasets. These @@ -83,13 +88,6 @@ feed. So edit the following section in `config/datasets.json` so that the `index ## Start ElasticSearch -By default, ElasticSearch starts with security restrictions configured in `config/elasticsearch.yml`. For production applications, disabling security is not recommended but for this example, these can be set to `false` rather than `true` as below: - -``` -# Enable security features -xpack.security.enabled: false -``` - We then need to startup ElasticSearch so we can configure some indexes to hold the data. @@ -101,7 +99,7 @@ rake es:start This just runs `./server/bin/elasticsearch` so you can run that directly if you prefer. -Make a note of the password for the 'elastic' user. +Make a note of the password for the 'elastic' user and the enrollement token for kibana. Test it is running by visiting `http://localhost:9200/`. Log in with the user 'elastic' and the password shown in the terminal log. You should see a JSON response from your local ElasticSearch server. From 42cc7af21f32fae8ae3020f1e60116d85fcf33c0 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Thu, 12 May 2022 15:30:39 +0100 Subject: [PATCH 13/21] Update Gemfile --- Gemfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 2ea1636..2154bee 100644 --- a/Gemfile +++ b/Gemfile @@ -2,10 +2,9 @@ source 'https://rubygems.org' ruby "2.6.10" gem 'json' -gem 'elasticsearch' +gem 'elasticsearch', '~> 8.2' gem 'rest-client' gem 'dotenv' gem 'openactive', :git => 'https://github.com/theodi/openactive.rb' - From 0f4b4018689b9df9ccad933240263602b491a737 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 14:45:31 +0100 Subject: [PATCH 14/21] Update README.md --- README.md | 51 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 47520b8..8d7aebc 100644 --- a/README.md +++ b/README.md @@ -53,11 +53,6 @@ server/logs server/plugins ``` -Go to the kibana download page and download the latest version for your platform. -Unzip this and move the contents into the kibana subfolder. - - - ## Update the list of datasets The ruby scripts that manage the indexes and harvest the data are driven off a list of published datasets. These @@ -97,20 +92,48 @@ We then need to startup ElasticSearch so we can configure some indexes to hold t rake es:start ``` -This just runs `./server/bin/elasticsearch` so you can run that directly if you prefer. +(This just runs `./server/bin/elasticsearch` so you can run that directly if you prefer.) + +Make a note of the password for the 'elastic' user and the HTTP CA certificate SHA-256 fingerprint in the output. You may have to scroll back up to see it. + +Edit the file `bin/connect.rb`. Paste in the password and certificate details from the console output and save this file. -Make a note of the password for the 'elastic' user and the enrollement token for kibana. +You can test the elasticsearch instance is running by visiting `https://localhost:9200/`. +You may see a certifcate error alert, depending upon your browser, which you can bypass e.g. click Advanced, then select the option to Proceed. + +Log in with the user 'elastic' and the password shown in the terminal log. + +If the instance is running, you should see a JSON response from your local ElasticSearch server, something like: + +``` +{ + "name" : "MacBook-Pro.local", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "XZB1wYYJTX3KNltDMVA", + "version" : { + "number" : "8.2.0", + "build_flavor" : "default", + "build_type" : "tar", + "build_hash" : "b174af62e8Sf4aDDc4d25gf875e93812b9282c5", + "build_date" : "2022-04-20T10:35:10.180408517Z", + "build_snapshot" : false, + "lucene_version" : "9.1.0", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" +} +``` -Test it is running by visiting `http://localhost:9200/`. Log in with the user 'elastic' and the password shown in the terminal log. You should see a JSON response from your local ElasticSearch server. -You can Ctrl-C to shutdown the server at any time. But it needs to be running for the following steps. +You can Ctrl-C to shutdown the server at any time - but it needs to be running for the following steps. -ElasticSearch is configured via its API so you need to have a running nstance available. +ElasticSearch is configured via its API so you need to have a irunning nstance available. ### Aside: Using a different ElasticSearch Server -The scripts all assume that they are working with an ElasticSearch instance available at `http://localhost:9200/`. +The scripts all assume that they are working with an ElasticSearch instance available at `https://localhost:9200/`. If you want to use an alternative server, then for the moment you'll need to edit the scripts in the `bin` directory to revise the following lines: @@ -206,14 +229,14 @@ We can now check to see that we have indexed some data. If you visit this URL: ``` -http://localhost:9200/_stats +https://localhost:9200/_stats ``` Then ElasticSearch will dump the current state of all indexes, including how many documents are in each. You can also ask for index specific statistics. So, assuming you have indexed British Triathlon, then if you visit this URL: ``` -http://localhost:9200/oa-britishtriathlon.github.io/_stats +https://localhost:9200/oa-britishtriathlon.github.io/_stats ``` Then you should get stats for just that index. You should see something like this: @@ -227,7 +250,7 @@ Which tells you that 1825 records have been indexed. You can also visit the `_search` endpoint to see all the docs: ``` -http://localhost:9200/oa-britishtriathlon.github.io/_search +https://localhost:9200/oa-britishtriathlon.github.io/_search ``` We suggest reading [the ElasticSearch documentation on their search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/_exploring_your_data.html) From 7765a910d393956bbccb01e19369a0901c2e1478 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 14:47:27 +0100 Subject: [PATCH 15/21] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d7aebc..a812808 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ This repository contains a simple demonstration of harvesting and indexing [opportunity data](http://status.openactive.io/), published as part of the [OpenActive](https://openactive.io) initiative. The example uses a set of simple Ruby scripts to drive harvesting of live data feeds and indexes the data in -[ElasticSearch](https://www.elastic.co/) and Kibana, open source search tools. +[ElasticSearch](https://www.elastic.co/), an open source search tool. The code in this project is published under an open licence and you are free to adapt and reuse it as you see fit. @@ -102,7 +102,7 @@ You can test the elasticsearch instance is running by visiting `https://localhos You may see a certifcate error alert, depending upon your browser, which you can bypass e.g. click Advanced, then select the option to Proceed. -Log in with the user 'elastic' and the password shown in the terminal log. +Log in with the user 'elastic' and the password noted earlier. If the instance is running, you should see a JSON response from your local ElasticSearch server, something like: From 019f52516109a27b9ee24fcf15968e13e58648b3 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 15:52:29 +0100 Subject: [PATCH 16/21] Create connect.rb --- bin/connect.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 bin/connect.rb diff --git a/bin/connect.rb b/bin/connect.rb new file mode 100644 index 0000000..0f6cf89 --- /dev/null +++ b/bin/connect.rb @@ -0,0 +1,6 @@ +# Colons and uppercase/lowercase don't matter when using +# the 'ca_fingerprint' parameter +CERT_FINGERPRINT = "XXXXXXXXYYYYYYXXXXXXXYYYYYYYYYXXXXXXXXYYYYYYYYYXXXXXXXXXXXX" + +# Password for the 'elastic' user generated by Elasticsearch +ELASTIC_PASSWORD = "XXXXXXXXXXXXXXXXXXXXXX" From 98cf243ea940f4dc99836bf724a3a0b7da937805 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 15:53:45 +0100 Subject: [PATCH 17/21] Update create-indexes.rb --- bin/create-indexes.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/create-indexes.rb b/bin/create-indexes.rb index 9ee5f2f..ecd97ee 100644 --- a/bin/create-indexes.rb +++ b/bin/create-indexes.rb @@ -4,7 +4,13 @@ INDEX_PREFIX="oa" -client = Elasticsearch::Client.new +require_relative 'connect' + +client = Elasticsearch::Client.new( + host: "https://elastic:#{ELASTIC_PASSWORD}@localhost:9200", + transport_options: { ssl: { verify: false } }, + ca_fingerprint: CERT_FINGERPRINT +) #Create index template, new templates will follow this pattern template = JSON.parse( File.read( ARGV[1] ) ) From 36dc06b4514c9f496b64c84212439df6ea403b8d Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 15:54:08 +0100 Subject: [PATCH 18/21] Update delete-index.rb --- bin/delete-index.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/delete-index.rb b/bin/delete-index.rb index 901c59d..6643a70 100644 --- a/bin/delete-index.rb +++ b/bin/delete-index.rb @@ -4,7 +4,13 @@ INDEX_PREFIX="oa" -client = Elasticsearch::Client.new +require_relative 'connect' + +client = Elasticsearch::Client.new( + host: "https://elastic:#{ELASTIC_PASSWORD}@localhost:9200", + transport_options: { ssl: { verify: false } }, + ca_fingerprint: CERT_FINGERPRINT +) index_name = "#{INDEX_PREFIX}-#{ARGV[0]}" From 352ed09f8ef336d7c962290796c1be3442c6b2c1 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 15:54:28 +0100 Subject: [PATCH 19/21] Update delete-indexes.rb --- bin/delete-indexes.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/delete-indexes.rb b/bin/delete-indexes.rb index 9d0db1a..cb33cbb 100644 --- a/bin/delete-indexes.rb +++ b/bin/delete-indexes.rb @@ -4,8 +4,13 @@ INDEX_PREFIX="oa" -client = Elasticsearch::Client.new +require_relative 'connect' +client = Elasticsearch::Client.new( + host: "https://elastic:#{ELASTIC_PASSWORD}@localhost:9200", + transport_options: { ssl: { verify: false } }, + ca_fingerprint: CERT_FINGERPRINT +) datasets = JSON.parse( File.read(ARGV[0]) ) datasets.keys.each do |dataset| From 0fc9862d869ab7578acc3a59ef06940539a7dec6 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Mon, 16 May 2022 15:54:53 +0100 Subject: [PATCH 20/21] Update harvest.rb --- bin/harvest.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/harvest.rb b/bin/harvest.rb index 7bf7146..0114269 100644 --- a/bin/harvest.rb +++ b/bin/harvest.rb @@ -2,7 +2,13 @@ require 'bundler' Bundler.require :default -client = Elasticsearch::Client.new +require_relative 'connect' + +client = Elasticsearch::Client.new( + host: "https://elastic:#{ELASTIC_PASSWORD}@localhost:9200", + transport_options: { ssl: { verify: false } }, + ca_fingerprint: CERT_FINGERPRINT +) #Parse the config/datasets.json file datasets = JSON.parse( File.read(ARGV[0]) ) From 2234a1c49c2058544b7d8b1b47153eae338b05b4 Mon Sep 17 00:00:00 2001 From: howaskew <19271569+howaskew@users.noreply.github.com> Date: Wed, 18 May 2022 16:05:45 +0100 Subject: [PATCH 21/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a812808..8a7d4d9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ cd openactive-es-example bundle install ``` -## Download and unzip Elastic Search and Kibana +## Download and unzip Elastic Search Go to [the Elastic Search download page](https://www.elastic.co/downloads/elasticsearch) and download the zip file of the latest release.