diff --git a/integration-tests/README.md b/integration-tests/README.md index f9e22ec..2f9c048 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -20,5 +20,5 @@ the integration data is loaded by referencing the [bestbets-loader](https://gith * [Docs for understanding how to run Karate standalone](https://github.com/intuit/karate/blob/6de466bdcf105d72450a40cf31b8adb5c043037d/karate-netty/README.md#standalone-jar) * Specifically this has to do with the magic naming of the logging config which is really why I am posting this here! * We have docker for dev testing because ES will no longer run on higher Java versions, this is the easiest way to get it up and running. -* .NET running locally on a Mac cannot talk to ES because of how NEST always uses the host name to connect to ES and ES exposes the Virtual Machine's hostname/IP that runs Linux on the Mac. +* .NET running locally on a Mac cannot talk to ES because of how the client always uses the host name to connect to ES and ES exposes the Virtual Machine's hostname/IP that runs Linux on the Mac. * You need to use the `--force-recreate` option to `docker-compose up` or run `docker-compose rm` after shutting down the cluster. If the elasticsearch container is not removed, it keeps its data, and any restarts will leave the cluster in a bad state. diff --git a/integration-tests/docker-bestbets-api/api/Dockerfile b/integration-tests/docker-bestbets-api/api/Dockerfile index ddb14d0..5e0e9f9 100644 --- a/integration-tests/docker-bestbets-api/api/Dockerfile +++ b/integration-tests/docker-bestbets-api/api/Dockerfile @@ -3,7 +3,7 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 # Get the NIH Root certificates and install them so we work on VPN # Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx -RUN mkdir /usr/local/share/ca-certificates/nih \ +RUN mkdir -p /usr/local/share/ca-certificates/nih \ && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \ && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \ && update-ca-certificates diff --git a/integration-tests/docker-bestbets-api/docker-compose.yml b/integration-tests/docker-bestbets-api/docker-compose.yml index 864ceaa..092a414 100644 --- a/integration-tests/docker-bestbets-api/docker-compose.yml +++ b/integration-tests/docker-bestbets-api/docker-compose.yml @@ -9,8 +9,11 @@ services: environment: - discovery.type=single-node - ES_JAVA_OPTS=-Xms750m -Xmx750m - # Turn off warnings about not using HTTPS. + # Turn security settings off for testing. (Allows http instead of https.) - xpack.security.enabled=false + - xpack.security.transport.ssl.enabled=false + - xpack.security.http.ssl.enabled=false + - http.host=0.0.0.0 ## These exposed ports are for debugging only. .NET + ## Docker + MacOS == bad scene. (.NET always wants to ## use the hosts name, and on a mac that is actually diff --git a/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile b/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile index 29e6fcc..830a814 100644 --- a/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile +++ b/integration-tests/docker-bestbets-api/elasticsearch/Dockerfile @@ -1,9 +1,12 @@ -FROM elasticsearch:7.17.28 +FROM elasticsearch:8.19.12 + +USER root # Get the NIH Root certificates and install them so we work on VPN # Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx -RUN mkdir /usr/local/share/ca-certificates/nih \ +RUN mkdir -p /usr/local/share/ca-certificates/nih \ && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \ && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \ && update-ca-certificates +USER elasticsearch diff --git a/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs b/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs index 838420e..5564f77 100644 --- a/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs +++ b/src/NCI.OCPL.Api.BestBets/Controllers/BestBetsController.cs @@ -8,9 +8,6 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using Nest; -using Elasticsearch.Net; - using NCI.OCPL.Api.Common; namespace NCI.OCPL.Api.BestBets.Controllers diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs index 1104408..703c671 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/IBestBetsDisplayService.cs @@ -1,4 +1,3 @@ - using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs index 1484076..2aadada 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/IHealthCheckService.cs @@ -1,5 +1,4 @@ - -using System.Threading.Tasks; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets { diff --git a/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs b/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs index f4da30e..67bef13 100644 --- a/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs +++ b/src/NCI.OCPL.Api.BestBets/Interfaces/ITokenAnalyzerService.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; +using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets { diff --git a/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs b/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs index e711d89..0f8f511 100644 --- a/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs +++ b/src/NCI.OCPL.Api.BestBets/Models/BestBetsCategoryDisplay.cs @@ -1,36 +1,34 @@ -using System; -using Nest; +using System.Text.Json.Serialization; namespace NCI.OCPL.Api.BestBets { /// /// Represents Display information about a Best Bet /// - [ElasticsearchType(RelationName = "categorydisplay")] public class BestBetsCategoryDisplay : IBestBetDisplay { /// /// Gets or sets the name of the category for this Best Bet Match /// - [Text(Name = "name")] + [JsonPropertyName("name")] public string Name { get; set; } /// /// Gets or sets the content ID of the category of this match /// - [Text(Name = "contentid")] + [JsonPropertyName("contentid")] public string ID { get; set; } /// /// Gets or sets the HTML for display for this category /// - [Text(Name = "content")] + [JsonPropertyName("content")] public string HTML { get; set; } /// /// Gets the weight of this category to determine ordering on display /// - [Text(Name = "weight")] + [JsonPropertyName("weight")] public int Weight { get; set; } /// diff --git a/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs b/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs index 6b157ae..03130e9 100644 --- a/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs +++ b/src/NCI.OCPL.Api.BestBets/Models/BestBetsMatch.cs @@ -1,56 +1,53 @@ -using System; - -using Nest; +using System.Text.Json.Serialization; namespace NCI.OCPL.Api.BestBets { /// /// Represents a Best Best Match from the Search Engine /// - [ElasticsearchType(RelationName = "synonyms")] public class BestBetsMatch { /// /// Gets or sets the name of the category for this Best Bet Match /// - [Text(Name = "category")] + [JsonPropertyName("category")] public string Category { get; set; } /// /// Gets or sets the content ID of the category of this match /// - [Text(Name = "contentid")] + [JsonPropertyName("contentid")] public string ContentID {get; set;} /// /// Gets or sets the synonym that was matched /// - [Text(Name = "synonym")] + [JsonPropertyName("synonym")] public string Synonym {get; set;} /// /// Gets or sets the two-character language code for this match /// - [Text(Name = "language")] + [JsonPropertyName("language")] public string Language {get; set;} /// /// Gets or sets whether this match is a negated one or not. /// - [Boolean(Name = "is_negated")] + [JsonPropertyName("is_negated")] public bool IsNegated {get; set;} /// /// Gets or sets whether this match is an exact one or not. /// - [Boolean(Name = "is_exact")] + [JsonPropertyName("is_exact")] public bool IsExact {get; set;} /// /// Gets or sets the number of tokens the ElasticSearch analyzer would /// return for this synonym. /// - [Number(Name = "tokencount")] + [JsonPropertyName("tokencount")] public int TokenCount {get; set;} } diff --git a/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj b/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj index 6bfe96a..f45af4b 100644 --- a/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj +++ b/src/NCI.OCPL.Api.BestBets/NCI.OCPL.Api.BestBets.csproj @@ -1,13 +1,12 @@ - netcoreapp8.0 + net8.0 true - - - + + diff --git a/src/NCI.OCPL.Api.BestBets/Program.cs b/src/NCI.OCPL.Api.BestBets/Program.cs index f767afa..84362a8 100644 --- a/src/NCI.OCPL.Api.BestBets/Program.cs +++ b/src/NCI.OCPL.Api.BestBets/Program.cs @@ -1,13 +1,4 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - - using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using NCI.OCPL.Api.Common; diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs index d4d2833..4f9bbd9 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsDisplayService.cs @@ -1,17 +1,10 @@ using System; -using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; - -using Newtonsoft.Json; - -using NCI.OCPL.Api.BestBets; +using Elastic.Clients.Elasticsearch; using NCI.OCPL.Api.Common; @@ -23,7 +16,7 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESBestBetsDisplayService : IBestBetsDisplayService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; @@ -33,7 +26,7 @@ public class ESBestBetsDisplayService : IBestBetsDisplayService /// The client to be used for connections /// The client to be used for connections /// The client to be used for connections - public ESBestBetsDisplayService(IElasticClient client, + public ESBestBetsDisplayService(ElasticsearchClient client, IOptions config, ILogger logger) { _elasticClient = client; @@ -81,18 +74,18 @@ public async Task GetBestBetForDisplay(string collection, strin } // If the API's response isn't valid, throw an error and return 500 status code. - if (!response.IsValid) + if (!response.IsValidResponse) { throw new APIErrorException(500, "Errors occurred."); } // If the API finds the category ID, return the resource. - if (response.Found && response.IsValid) + if (response.Found && response.IsValidResponse) { result = response.Source; } // If the API cannot find the category ID, throw an error and return 404 status code. - else if (!response.Found && response.IsValid) + else if (!response.Found && response.IsValidResponse) { throw new APIErrorException(404, "Category not found."); } diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs index ccfa1a8..9013dd7 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsHealthService.cs @@ -1,17 +1,12 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; - -using Newtonsoft.Json; - -using NCI.OCPL.Api.BestBets; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Cluster; using NCI.OCPL.Api.Common; @@ -23,7 +18,7 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESBestBetsHealthService : IHealthCheckService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; @@ -33,7 +28,7 @@ public class ESBestBetsHealthService : IHealthCheckService /// The client to be used for connections /// The client to be used for connections /// The client to be used for connections - public ESBestBetsHealthService(IElasticClient client, + public ESBestBetsHealthService(ElasticsearchClient client, IOptions config, ILogger logger) { _elasticClient = client; @@ -75,17 +70,16 @@ private async Task IsHostHealthy(string alias) try { - //ClusterHealthResponse response = await _elasticClient.Cluster.HealthAsync(hd => hd.Index(alias)); - ClusterHealthResponse response = await _elasticClient.Cluster.HealthAsync(null, hd=> hd.Index(alias)); + HealthResponse response = await _elasticClient.Cluster.HealthAsync(new HealthRequest(alias)); - if (!response.IsValid) + if (!response.IsValidResponse) { _logger.LogError($"Error checking ElasticSearch health for {alias}."); - _logger.LogError($"Returned debug info: {response.DebugInformation}."); + _logger.LogError("Returned error reason: {reason}.", response.ElasticsearchServerError?.Error?.Reason); } else { - if (response.Status == Health.Green || response.Status == Health.Yellow) + if (response.Status == HealthStatus.Green || response.Status == HealthStatus.Yellow) { //This is the only condition that will return true return true; diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs index fe64e78..5a6a756 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESBestBetsMatchService.cs @@ -6,8 +6,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using NCI.OCPL.Api.Common; @@ -21,7 +21,7 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESBestBetsMatchService : IBestBetsMatchService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private ITokenAnalyzerService _tokenAnalyzer; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; @@ -29,7 +29,7 @@ public class ESBestBetsMatchService : IBestBetsMatchService /// /// Creates a new instance of a ESBestBetsMatchService /// - public ESBestBetsMatchService(IElasticClient client, + public ESBestBetsMatchService(ElasticsearchClient client, ITokenAnalyzerService tokenAnalyzer, IOptions config, ILogger logger) @@ -137,21 +137,37 @@ private string[] FilterValidCategories(IEnumerable matches, int n private async Task> GetSetOfMatchesAsync(string collection, string cleanedTerm, int searchTokenCount, string lang, int matchedTokenCount) { //This is the query - var matchQuery = new NumericRangeQuery { Field = "tokencount", LessThanOrEqualTo = matchedTokenCount } && - new TermQuery { Field = "is_exact", Value = false } && - new TermQuery { Field = "language", Value = lang } && - new MatchQuery { Field = "synonym", Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount } && - new TermQuery { Field = "record_type", Value = "synonyms" }; + var mustClauses = new List + { + new NumberRangeQuery(new Field("tokencount")) { Lte = matchedTokenCount }, + new TermQuery { Field = new Field("is_exact"), Value = false }, + new TermQuery { Field = new Field("language"), Value = lang }, + new MatchQuery { Field = new Field("synonym"), Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount.ToString() }, + new TermQuery { Field = new Field("record_type"), Value = "synonyms" } + }; + + Query matchQuery = new BoolQuery { Must = mustClauses }; if (searchTokenCount == matchedTokenCount) { //Add in exact match query too - matchQuery = matchQuery || - new TermQuery { Field = "tokencount", Value = matchedTokenCount } && - new TermQuery { Field = "is_exact", Value = true } && - new TermQuery { Field = "language", Value = lang } && - new MatchQuery { Field = "synonym", Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount } && - new TermQuery { Field = "record_type", Value = "synonyms"}; + var exactClauses = new List + { + new TermQuery { Field = new Field("tokencount"), Value = matchedTokenCount }, + new TermQuery { Field = new Field("is_exact"), Value = true }, + new TermQuery { Field = new Field("language"), Value = lang }, + new MatchQuery { Field = new Field("synonym"), Query = cleanedTerm, MinimumShouldMatch = matchedTokenCount.ToString() }, + new TermQuery { Field = new Field("record_type"), Value = "synonyms" } + }; + + matchQuery = new BoolQuery + { + Should = new List + { + matchQuery, + new BoolQuery { Must = exactClauses } + } + }; } try @@ -160,7 +176,7 @@ private async Task> GetSetOfMatchesAsync(string colle this._bestbetsConfig.PreviewAliasName : this._bestbetsConfig.LiveAliasName; - var req = new SearchRequest(alias) + var req = new SearchRequest(alias) { Query = matchQuery, Size = 10000 //Make sure this more than the number of synonyms @@ -169,14 +185,14 @@ private async Task> GetSetOfMatchesAsync(string colle var response = await this._elasticClient.SearchAsync(req); //Test if response is valid - if (!response.IsValid) + if (!response.IsValidResponse) { _logger.LogError("Elasticsearch Response is Not Valid. Term '{0}'", cleanedTerm.Replace(Environment.NewLine, String.Empty)); - _logger.LogError("Returned debug info: {0}.", response.DebugInformation); + _logger.LogError("Returned error reason: {0}.", response.ElasticsearchServerError?.Error?.Reason); throw new APIErrorException(500, "Errors Occurred."); } - return response.Documents; + return response.Hits.Select(h => h.Source).Where(s => s != null)!; } catch (APIErrorException) diff --git a/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs b/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs index 0d97495..af38268 100644 --- a/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs +++ b/src/NCI.OCPL.Api.BestBets/Services/ESTokenAnalyzerService.cs @@ -1,13 +1,12 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Elasticsearch.Net; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; using NCI.OCPL.Api.Common; @@ -20,14 +19,14 @@ namespace NCI.OCPL.Api.BestBets.Services /// public class ESTokenAnalyzerService : ITokenAnalyzerService { - private IElasticClient _elasticClient; + private ElasticsearchClient _elasticClient; private CGBBIndexOptions _bestbetsConfig; private readonly ILogger _logger; /// /// Creates a new instance of a ESBestBetsMatchService /// - public ESTokenAnalyzerService(IElasticClient client, + public ESTokenAnalyzerService(ElasticsearchClient client, IOptions bestbetsConfig, ILogger logger) //Needs someway to get an IElasticClient { @@ -46,7 +45,7 @@ public async Task GetTokenCount(string collection, string term) { string[] ALLOWED_TOKEN_TYPES = { "", ""}; - AnalyzeResponse analyzeResponse; + AnalyzeIndexResponse analyzeResponse; string indexForAnalysis = (collection == "preview") ? _bestbetsConfig.PreviewAliasName : _bestbetsConfig.LiveAliasName; @@ -54,32 +53,32 @@ public async Task GetTokenCount(string collection, string term) try { analyzeResponse = await this._elasticClient.Indices.AnalyzeAsync( - a => a - .Index(indexForAnalysis) - .Analyzer("nostem") - .Text(term) + new AnalyzeIndexRequest(indexForAnalysis) + { + Analyzer = "nostem", + Text = new[] { term } + } ); } - catch(UnexpectedElasticsearchClientException ex) + catch(Exception ex) { - _logger.LogError(ex, "Error analyzing token count for term '{0}'. Reason: '{1}'. DebugInformation: {2}", - term.Replace(Environment.NewLine, String.Empty), - ex.FailureReason, ex.DebugInformation); + _logger.LogError(ex, "Error analyzing token count for term '{0}'.", term.Replace(Environment.NewLine, String.Empty)); _logger.LogInformation("Trying again for term '{0}", term); // Try again. (this is really just for when we run out of sockets) analyzeResponse = await this._elasticClient.Indices.AnalyzeAsync( - a => a - .Index(indexForAnalysis) - .Analyzer("nostem") - .Text(term) + new AnalyzeIndexRequest(indexForAnalysis) + { + Analyzer = "nostem", + Text = new[] { term } + } ); } - if (!analyzeResponse.IsValid) + if (!analyzeResponse.IsValidResponse) { _logger.LogError("Elasticsearch Response for GetTokenCount is Not Valid. Term '{0}'", term); - _logger.LogError("Returned debug info: {0}.", analyzeResponse.DebugInformation); + _logger.LogError("Returned error reason: {0}.", analyzeResponse.ElasticsearchServerError?.Error?.Reason); throw new APIErrorException(500, "Errors Occurred."); } diff --git a/src/NCI.OCPL.Api.BestBets/Startup.cs b/src/NCI.OCPL.Api.BestBets/Startup.cs index 95b45ef..2bea70f 100644 --- a/src/NCI.OCPL.Api.BestBets/Startup.cs +++ b/src/NCI.OCPL.Api.BestBets/Startup.cs @@ -1,26 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -using Nest; -using Elasticsearch.Net; - -using NSwag.AspNetCore; -using NJsonSchema; -using System.Reflection; -using NCI.OCPL.Api.Common; using NCI.OCPL.Api.BestBets.Services; +using NCI.OCPL.Api.Common; namespace NCI.OCPL.Api.BestBets { diff --git a/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj b/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj index 3208a2d..f350e1d 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj +++ b/test/NCI.OCPL.Api.BestBets.Tests/NCI.OCPL.Api.BestBets.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp8.0 + net8.0 @@ -15,16 +15,19 @@ - + - - - + + + + + + diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs index 01cac7e..d6fd279 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Controllers/BestBetsControllerTests.cs @@ -1,25 +1,14 @@ -using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Testing; -using Elasticsearch.Net; -using Nest; - -using Newtonsoft.Json.Linq; - using Moq; using Xunit; using NCI.OCPL.Api.Common; -using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.BestBets; using NCI.OCPL.Api.BestBets.Controllers; -using NCI.OCPL.Api.BestBets.Tests.ESHealthTestData; using NCI.OCPL.Api.BestBets.Tests.ESDisplayTestData; namespace NCI.OCPL.Api.BestBets.Tests diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs index f472e50..d246f7f 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/ArrayComparer.cs @@ -1,7 +1,6 @@ -using System; -using System.Linq; -using System.Collections.Generic; using System.Collections; +using System.Collections.Generic; +using System.Linq; namespace NCI.OCPL.Api.BestBets.Tests { @@ -19,7 +18,7 @@ public class ArrayComparer : IEqualityComparer /// public bool Equals(string[] x, string[] y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. if (x == null && y == null) diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs index 78ef1dc..2139d16 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetCategoryComparer.cs @@ -1,7 +1,6 @@ -using System; -using System.Linq; -using System.Collections.Generic; using System.Collections; +using System.Collections.Generic; +using System.Linq; namespace NCI.OCPL.Api.BestBets.Tests { @@ -14,12 +13,12 @@ public class IBestBetSynonymComparer : IEqualityComparer public bool Equals(IBestBetSynonym x, IBestBetSynonym y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. - if (x == null && y== null) + if (x == null && y== null) { return true; - } + } else if (x == null || y == null) { return false; @@ -28,7 +27,7 @@ public bool Equals(IBestBetSynonym x, IBestBetSynonym y) bool isEqual = x.IsExactMatch == y.IsExactMatch && x.Text == y.Text; - + return isEqual; } @@ -52,27 +51,27 @@ public class IBestBetCategoryComparer : IEqualityComparer public bool Equals(IBestBetCategory x, IBestBetCategory y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. - if (x == null && y== null) + if (x == null && y== null) { return true; - } + } else if (x == null || y == null) { return false; - } + } - bool isEqual = + bool isEqual = _displayComparer.Equals(x,y) //Handles ID, Name, Weight, HTML && x.Display == y.Display && x.IsExactMatch == y.IsExactMatch && x.Language == y.Language && AreSynonymListsEqual(x.ExcludeSynonyms, y.ExcludeSynonyms) && AreSynonymListsEqual(x.IncludeSynonyms, y.IncludeSynonyms); - - - return isEqual; + + + return isEqual; } /// @@ -82,17 +81,17 @@ public bool Equals(IBestBetCategory x, IBestBetCategory y) /// Synonym list 2 /// private bool AreSynonymListsEqual(IBestBetSynonym[] x, IBestBetSynonym[] y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. - - if (x == null && y== null) + + if (x == null && y== null) { return true; - } + } else if (x == null || y == null) { return false; - } + } //Generate a set of those values that are not in both lists. //if this is not 0, then there is an error. @@ -104,7 +103,7 @@ private bool AreSynonymListsEqual(IBestBetSynonym[] x, IBestBetSynonym[] y) { public int GetHashCode(IBestBetCategory obj) { int hash = 0; - hash ^= + hash ^= _displayComparer.GetHashCode(obj) //Handles ID, Name, Weight, HTML ^ obj.Display.GetHashCode() ^ obj.Language.GetHashCode() @@ -114,6 +113,6 @@ public int GetHashCode(IBestBetCategory obj) return hash; } - + } } \ No newline at end of file diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs index f84a3d7..1a13ce7 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Models/IBestBetDisplayComparer.cs @@ -1,7 +1,6 @@ -using System; -using System.Linq; -using System.Collections.Generic; using System.Collections; +using System.Collections.Generic; +using System.Linq; namespace NCI.OCPL.Api.BestBets.Tests { @@ -13,7 +12,7 @@ public class IBestBetDisplayComparer : IEqualityComparer { public bool Equals(IBestBetDisplay x, IBestBetDisplay y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. if (x == null && y == null) { @@ -65,7 +64,7 @@ public int GetHashCode(IBestBetDisplay obj) /// private bool AreParamArraysEqual(string[] x, string[] y) { - // If the items are both null, or if one or the other is null, return + // If the items are both null, or if one or the other is null, return // the correct response right away. if (x == null && y == null) diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs index c259736..ebb797c 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsDisplayServiceTests.cs @@ -1,21 +1,16 @@ using System; using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Text; -using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; -using Xunit; +using Elastic.Clients.Elasticsearch; using Moq; -using RichardSzalay.MockHttp; +using Xunit; -using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.Common; +using NCI.OCPL.Api.Common.Testing; -using Elasticsearch.Net; -using Nest; using NCI.OCPL.Api.BestBets.Services; using NCI.OCPL.Api.BestBets.Tests.ESDisplayTestData; @@ -41,24 +36,13 @@ public class GetBestBetForDisplayTests public async void GetBestBetForDisplay_TestURISetup(BaseDisplayTestData data) { Uri esURI = null; - - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream("ESDisplayData/" + data.TestFilePath); - - res.StatusCode = 200; - - esURI = req.Uri; - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + string responseBody = TestingTools.ReadTestFile("ESDisplayData/" + data.TestFilePath); + var settings = TestingElasticsearchClientSettingsFactory.Create( + responseBody, + 200, + details => esURI = details.Uri + ); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -88,18 +72,8 @@ public async void GetBestBetForDisplay_TestURISetup(BaseDisplayTestData data) [Fact()] public async void GetBestBetForDisplay_TestAPIConnectionFailure() { - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - res.StatusCode = 500; - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + var settings = TestingElasticsearchClientSettingsFactory.Create("{}", 500); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -116,18 +90,8 @@ public async void GetBestBetForDisplay_TestAPIConnectionFailure() [Fact()] public async void GetBestBetForDisplay_TestInvalidResponse() { - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + var settings = TestingElasticsearchClientSettingsFactory.Create("not-json", 200); + ElasticsearchClient client = new ElasticsearchClient(settings); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -146,7 +110,7 @@ public async void GetBestBetForDisplay_TestInvalidResponse() [Theory, MemberData(nameof(JsonData))] public async void GetBestBetForDisplay_DataLoading(BaseDisplayTestData data) { - IElasticClient client = GetElasticClientWithData(data); + ElasticsearchClient client = GetElasticClientWithData(data); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -164,7 +128,7 @@ public async void GetBestBetForDisplay_DataLoading(BaseDisplayTestData data) [Theory, MemberData(nameof(NotFoundData))] public async void GetBestBetForDisplay_IDNotFoundError(BaseDisplayTestData data) { - IElasticClient client = GetElasticClientWithData(data); + ElasticsearchClient client = GetElasticClientWithData(data); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -181,7 +145,7 @@ public async void GetBestBetForDisplay_IDNotFoundError(BaseDisplayTestData data) [Theory, MemberData(nameof(JsonData))] public async void GetBestBetForDisplay_InvalidIDError(BaseDisplayTestData data) { - IElasticClient client = GetElasticClientWithData(data); + ElasticsearchClient client = GetElasticClientWithData(data); // Setup the mocked Options IOptions bbClientOptions = GetMockOptions(); @@ -192,24 +156,10 @@ public async void GetBestBetForDisplay_InvalidIDError(BaseDisplayTestData data) Assert.Equal(400, ex.HttpStatusCode); } - private IElasticClient GetElasticClientWithData(BaseDisplayTestData data) { - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - conn.RegisterRequestHandlerForType>((req, res) => - { - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream("ESDisplayData/" + data.TestFilePath); - - res.StatusCode = 200; - }); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); - - return client; + private ElasticsearchClient GetElasticClientWithData(BaseDisplayTestData data) { + string responseBody = TestingTools.ReadTestFile("ESDisplayData/" + data.TestFilePath); + var settings = TestingElasticsearchClientSettingsFactory.Create(responseBody, 200); + return new ElasticsearchClient(settings); } private IOptions GetMockOptions() diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs index c160d76..19d7ae4 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsHealthServiceTests.cs @@ -1,16 +1,11 @@ -using System; -using System.Collections.Generic; - -using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; -using Nest; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; using Xunit; +using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.BestBets.Services; -using NCI.OCPL.Api.BestBets.Tests.ESHealthTestData; -using NCI.OCPL.Api.BestBets.Tests.ESMatchTestData; namespace NCI.OCPL.Api.BestBets.Tests { @@ -21,9 +16,7 @@ public class ESBestBetsHealthServiceTests : TestServiceBase [InlineData("yellow")] public async void HealthStatus_Healthy(string datafile) { - ESHealthConnection connection = new ESHealthConnection(datafile); - - ESBestBetsHealthService service = GetHealthService(connection); + ESBestBetsHealthService service = GetHealthServiceFromFile($"ESHealthData/{datafile}.json", 200); bool isHealthy = await service.IsHealthy(); @@ -35,8 +28,7 @@ public async void HealthStatus_Healthy(string datafile) [InlineData("unexpected")] // i.e. "Unexpected color" public async void HealthStatus_Unhealthy(string datafile) { - ESHealthConnection connection = new ESHealthConnection(datafile); - ESBestBetsHealthService service = GetHealthService(connection); + ESBestBetsHealthService service = GetHealthServiceFromFile($"ESHealthData/{datafile}.json", 200); bool isHealthy = await service.IsHealthy(); @@ -53,22 +45,22 @@ public async void HealthStatus_Unhealthy(string datafile) [InlineData(500)] public async void HealthStatus_InvalidResponse(int httpStatus) { - ESErrorConnection connection = new ESErrorConnection(httpStatus); - ESBestBetsHealthService service = GetHealthService(connection); + ESBestBetsHealthService service = GetHealthService("{}", httpStatus); bool res = await service.IsHealthy(); Assert.False(res); - } - private ESBestBetsHealthService GetHealthService(IConnection connection) + private ESBestBetsHealthService GetHealthServiceFromFile(string filename, int statusCode) { - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + string responseBody = TestingTools.ReadTestFile(filename); + return GetHealthService(responseBody, statusCode); + } - var connectionSettings = new ConnectionSettings(pool, connection); - IElasticClient client = new ElasticClient(connectionSettings); + private ESBestBetsHealthService GetHealthService(string responseBody, int statusCode) + { + var settings = TestingElasticsearchClientSettingsFactory.Create(responseBody, statusCode); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs index baa12b8..e7b702b 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESBestBetsMatchServiceTests.cs @@ -1,30 +1,27 @@ -using System; using System.Collections.Generic; +using System.Text.Json.Nodes; using Microsoft.Extensions.Options; using Microsoft.Extensions.Logging.Testing; -using Nest; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Xunit; using NCI.OCPL.Api.BestBets.Services; -using NCI.OCPL.Api.BestBets.Tests.ESHealthTestData; -using NCI.OCPL.Api.BestBets.Tests.ESMatchTestData; +using NCI.OCPL.Api.Common.Testing; namespace NCI.OCPL.Api.BestBets.Tests { public class ESBestBetsMatchServiceTests : TestServiceBase { - public static IEnumerable GetMatchesData => new[] { // "pancoast" is a simple test as it only has 1 hit, 1 word, and 1 BB category. new object[] { "pancoast", "en", - new ESMatchConnection("pancoast"), - new ESMatchTokenizerConnection("pancoast"), + "pancoast", new string[] { "36012" } }, // "breast cancer" is more complicated, it has 1 hit, 2 words, and the BB category @@ -32,8 +29,7 @@ public class ESBestBetsMatchServiceTests : TestServiceBase new object[] { "breast cancer", "en", - new ESMatchConnection("breastcancer"), - new ESMatchTokenizerConnection("breastcancer"), + "breastcancer", new string[] { "36408" } }, // "breast cancer treatment" is more complicated, it has 1 hit, 3 words, and no results for last page. @@ -41,16 +37,14 @@ public class ESBestBetsMatchServiceTests : TestServiceBase new object[] { "breast cancer treatment", "en", - new ESMatchConnection("breastcancertreatment"), - new ESMatchTokenizerConnection("breastcancertreatment"), + "breastcancertreatment", new string[] { "36408" } }, // "seer stat" is a negated exact match test. SEER should not be returned new object[] { "seer stat", "en", - new ESMatchConnection("seerstat"), - new ESMatchTokenizerConnection("seerstat"), + "seerstat", new string[] { } }, // "seer stat fact sheet" is a test to make sure the "seer stat" exact match is not hit because @@ -58,8 +52,7 @@ public class ESBestBetsMatchServiceTests : TestServiceBase new object[] { "seer stat fact sheet", "en", - new ESMatchConnection("seerstatfactsheet"), - new ESMatchTokenizerConnection("seerstatfactsheet"), + "seerstatfactsheet", new string[] { "36681" } } }; @@ -69,51 +62,68 @@ public class ESBestBetsMatchServiceTests : TestServiceBase public async void GetMatches_Normal( string searchTerm, string lang, - ESMatchConnection connection, - ESMatchTokenizerConnection tokenizerConn, + string responseFileBase, string[] expectedCategories ) { //Use real ES client, with mocked connection. - - ESTokenAnalyzerService tokenService = GetTokenizerService(tokenizerConn); - ESBestBetsMatchService service = GetMatchService(tokenService, connection); + ESTokenAnalyzerService tokenService = GetTokenizerService(responseFileBase); + ESBestBetsMatchService service = GetMatchService(tokenService, responseFileBase); string[] actualMatches = await service.GetMatches("live", lang, searchTerm); Assert.Equal(expectedCategories, actualMatches); } - private ESTokenAnalyzerService GetTokenizerService(IConnection connection) + private ESTokenAnalyzerService GetTokenizerService(string responseFileBase) { - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, connection); - IElasticClient client = new ElasticClient(connectionSettings); + string body = TestingTools.ReadTestFile($"ESMatchData/{responseFileBase}_analyze.json"); + var settings = TestingElasticsearchClientSettingsFactory.Create(body, 200); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); return new ESTokenAnalyzerService(client, config, new NullLogger()); } - private ESBestBetsMatchService GetMatchService(ESTokenAnalyzerService tokenService, IConnection connection) + private ESBestBetsMatchService GetMatchService(ESTokenAnalyzerService tokenService, string responseFileBase) { - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + string builder(PostData postData, JsonNode jsonBody) { + //Determine which round we are performing + //int numTokens = postObj["params"].matchedtokencount; + + //If this is one item the bool node will be the nested match + //if it is both exact and matches, then the bool node will + //be a should. This code is tightly matched to the built query + //in the implementation + int numTokens = -1; + + string value; + var boolNode = jsonBody["query"]["bool"]; + if (boolNode["should"] != null) { + value = boolNode["should"][0] + ["bool"]["must"][3] + ["match"]["synonym"] + ["minimum_should_match"].GetValue(); + } else { + value = boolNode["must"][3] + ["match"]["synonym"] + ["minimum_should_match"].GetValue(); + + } + numTokens = int.Parse(value); + + return TestingTools.ReadTestFile($"ESMatchData/{responseFileBase}_{numTokens}.json"); + } - var connectionSettings = new ConnectionSettings(pool, connection); - IElasticClient client = new ElasticClient(connectionSettings); + var invoker = new DynamicInMemoryConnection(builder); + var settings = TestingElasticsearchClientSettingsFactory.Create(invoker); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); return new ESBestBetsMatchService(client, tokenService, config, new NullLogger()); } - - - } -} \ No newline at end of file +} diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs index 80ab2e6..6b41b69 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/ESTokenAnalyzerServiceTests.cs @@ -1,20 +1,20 @@ -using System; +#nullable enable +using System; using System.Collections.Generic; -using System.IO; -using System.Text; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Microsoft.Extensions.Logging.Testing; using Microsoft.Extensions.Options; -using Nest; -using Elasticsearch.Net; -using Newtonsoft.Json.Linq; using Xunit; using NCI.OCPL.Api.Common.Testing; using NCI.OCPL.Api.BestBets.Services; - namespace NCI.OCPL.Api.BestBets.Tests { public class ESTokenAnalyzerServiceTests : TestServiceBase @@ -55,27 +55,13 @@ public async void GetTokenCount_Responses( int expectedCount ) { + JsonObject resObject = new JsonObject(); + resObject["tokens"] = new JsonArray(responseTokens + .Select(responseToken => JsonSerializer.SerializeToNode(responseToken)) + .ToArray()); - ElasticsearchInterceptingConnection conn = new ElasticsearchInterceptingConnection(); - - conn.RegisterRequestHandlerForType( - (req, res) => - { - JObject resObject = new JObject(); - resObject["tokens"] = JArray.FromObject(responseTokens); - byte[] byteArray = Encoding.UTF8.GetBytes(resObject.ToString()); - - res.Stream = new MemoryStream(byteArray); - res.StatusCode = 200; - } - ); - - //While this has a URI, it does not matter, an InMemoryConnection never requests - //from the server. - var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - - var connectionSettings = new ConnectionSettings(pool, conn); - IElasticClient client = new ElasticClient(connectionSettings); + var settings = TestingElasticsearchClientSettingsFactory.Create(resObject.ToString(), 200); + ElasticsearchClient client = new ElasticsearchClient(settings); IOptions config = GetMockConfig(); diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs index 3ec1c07..b1cf2f8 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Services/TestServiceBase.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -using Microsoft.Extensions.Options; +using Microsoft.Extensions.Options; using Moq; - - namespace NCI.OCPL.Api.BestBets.Tests { /// diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESErrorTestObjects/ESErrorConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESErrorTestObjects/ESErrorConnection.cs deleted file mode 100644 index 4ee25bb..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESErrorTestObjects/ESErrorConnection.cs +++ /dev/null @@ -1,29 +0,0 @@ -using NCI.OCPL.Api.Common.Testing; - -namespace NCI.OCPL.Api.BestBets.Tests.ESHealthTestData -{ - /// - /// Class used for mocking requests to Elasticsearch that return an error. - /// - /// - public class ESErrorConnection : ElasticsearchInterceptingConnection - { - /// - /// Creates a new instance of the ESMatchConnection class - /// - /// HTTP status code to return - public ESErrorConnection(int testErrorCode) - { - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - // Health check is a GET request (e.g. https://localhost:9299/_cluster/health/bestbets?pretty) - // but for an error, all we care about is the response code, so we don't load a data file. - - res.StatusCode = testErrorCode; - }); - - } - } -} - diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthConnection.cs deleted file mode 100644 index b29c65e..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthConnection.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -using NCI.OCPL.Api.Common.Testing; - -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESHealthTestData -{ - /// - /// Class used for mocking BestBet HealthCheck requests to Elasticsearch. This should be - /// used as the base class of test specific Connections object passed into an ElasticClient. - /// - /// - public class ESHealthConnection : ElasticsearchInterceptingConnection - { - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchConnection class - /// - /// The prefix of the test files - public ESHealthConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - // Health check is a GET request (e.g. https://localhost:9299/_cluster/health/bestbets?pretty) - // so we don't need to do anything special, just load the data file. - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName()); - - res.StatusCode = 200; - }); - - } - - private string GetTestFileName() - { - return string.Format("ESHealthData/{0}.json", TestFilePrefix); - } - } -} - diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthTokenizerConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthTokenizerConnection.cs deleted file mode 100644 index 5aa7576..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESHealthTestObjects/ESHealthTokenizerConnection.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -using NCI.OCPL.Api.Common.Testing; -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESHealthTestData -{ - /// - /// This class is a helper around the ElasticsearchInterceptingConntection to handle - /// analyzer responses for a BB HealthCheck Request. (Basically, this class only exists - /// because we need to have a TokenizerConnection in order to instantiate Match Service.) - /// - /// - public class ESHealthTokenizerConnection : ElasticsearchInterceptingConnection - { - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchTokenizerConnection class - /// - /// The prefix of the test files - public ESHealthTokenizerConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - //I don't care about the request for this... for now. - - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName()); - - res.StatusCode = 200; - }); - } - - private string GetTestFileName() - { - return string.Format("ESHealthData/{0}_analyze.json", TestFilePrefix); - } - - } -} diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchConnection.cs deleted file mode 100644 index bf904df..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchConnection.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System; -using System.IO; -using System.Text; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - - -using NCI.OCPL.Api.Common.Testing; -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESMatchTestData -{ - /// - /// Class used for mocking BestBet Match requests to Elasticsearch. This should be - /// used as the base class of test specific Connections object passed into an ElasticClient. - /// - /// - public class ESMatchConnection : ElasticsearchInterceptingConnection - { - - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchConnection class - /// - /// The prefix of the test files - public ESMatchConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType>((req, res) => - { - //Get the request parameters - dynamic postObj = this.GetRequestPost(req); - - //Determine which round we are performing - //int numTokens = postObj["params"].matchedtokencount; - - //If this is one item the bool node will be the nested match - //if it is both exact and matches, then the bool node will - //be a should. This code is tightly matched to the built query - //in the implementation - int numTokens = -1; - - var boolNode = postObj["query"]["bool"]; - if (boolNode["should"] != null) { - numTokens = boolNode["should"][0] - ["bool"]["must"][3] - ["match"]["synonym"] - ["minimum_should_match"].ToObject(); - } else { - numTokens = boolNode["must"][3] - ["match"]["synonym"] - ["minimum_should_match"].ToObject(); - } - - - - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName(numTokens)); - - res.StatusCode = 200; - }); - - } - - private string GetTestFileName(int numTerms) - { - return string.Format("ESMatchData/{0}_{1}.json", TestFilePrefix, numTerms); - } - } -} \ No newline at end of file diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchTokenizerConnection.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchTokenizerConnection.cs deleted file mode 100644 index 1a4881a..0000000 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/TestDataObjects/ESMatchTestObjects/ESMatchTokenizerConnection.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Elasticsearch.Net; - -using System.Text; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -using NCI.OCPL.Api.Common.Testing; -using NCI.OCPL.Api.BestBets.Tests.Util; - -namespace NCI.OCPL.Api.BestBets.Tests.ESMatchTestData -{ - /// - /// This class is a helper around the ElasticsearchInterceptingConntection to handle - /// analyzer responses for a BB Match Request - /// - /// - public class ESMatchTokenizerConnection : ElasticsearchInterceptingConnection - { - /// - /// Gets the prefix of a testdata file for this test. - /// - /// - private string TestFilePrefix { get; set; } - - /// - /// Creates a new instance of the ESMatchTokenizerConnection class - /// - /// The prefix of the test files - public ESMatchTokenizerConnection(string testFilePrefix) - { - this.TestFilePrefix = testFilePrefix; - - //Add Handlers - this.RegisterRequestHandlerForType((req, res) => - { - //I don't care about the request for this... for now. - - //Get the file name for this round - res.Stream = TestingTools.GetTestFileAsStream(GetTestFileName()); - - res.StatusCode = 200; - }); - } - - private string GetTestFileName() - { - return string.Format("ESMatchData/{0}_analyze.json", TestFilePrefix); - } - - } -} diff --git a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs index e7c9383..49774ab 100644 --- a/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs +++ b/test/NCI.OCPL.Api.BestBets.Tests/Tests/Util/BestBetsMatchComparer.cs @@ -1,7 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; namespace NCI.OCPL.Api.BestBets.Tests.Util {