diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java index 404712ac36..368fb9d2c1 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java @@ -566,7 +566,7 @@ eventId, new Date(), currentUser.getGivenName(), if (schoolId != null) { School school = schoolListReader.findSchoolById(schoolId); if (null != school) { - resultRow.add(school.getName()); + resultRow.add(school.getSchoolName()); } else { resultRow.add(schoolId); } diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java index 7190885c96..20f1d45c5d 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java @@ -20,16 +20,20 @@ * */ public class School { - private String urn; - private String name; - private String postcode; + private String schoolId; + private String countryCode; + private String schoolName; + private String town; + private String postalCode; + private Boolean excluded; private Boolean closed; /** * Enum to represent where this school object was created. */ public enum SchoolDataSource { - GOVERNMENT_UK, GOVERNMENT_IE, GOVERNMENT_SCO, GOVERNMENT_WAL, GOVERNMENT_NI, USER_ENTERED; + GOVERNMENT_UK, GOVERNMENT_SCT, GOVERNMENT_SCT_COL, GOVERNMENT_SCT_IND, GOVERNMENT_WLS, GOVERNMENT_WLS_FE, + GOVERNMENT_NIR, GOVERNMENT_IE, USER_ENTERED; @Override public String toString() { @@ -48,90 +52,156 @@ public School() { /** * Full constructor. - * - * @param urn - * - unique id - * @param name - * - name of the school. - * @param postcode - * -postcode of the school - * @param dataSource - * -dataSource of this information + * + * @param schoolId + * - unique school ID + * @param countryCode + * - country code for the school + * @param schoolName + * - name of the school + * @param town + * - name of the town where the school is located + * @param postalCode + * - postal code of the school + * @param excluded + * - whether the school is excluded when searching for schools by name * @param closed * - whether the school is closed + * @param dataSource + * - data source of this information */ - public School(final String urn, final String name, final String postcode, final Boolean closed, final SchoolDataSource dataSource) { - this.urn = urn; - this.name = name; - this.postcode = postcode; - this.dataSource = dataSource; + public School(final String schoolId, final String countryCode, final String schoolName, final String town, + final String postalCode, final Boolean excluded, final Boolean closed, final SchoolDataSource dataSource) { + this.schoolId = schoolId; + this.countryCode = countryCode; + this.schoolName = schoolName; + this.town = town; + this.postalCode = postalCode; + this.excluded = excluded; this.closed = closed; + this.dataSource = dataSource; } /** - * Gets the urn. - * - * @return the urn + * Gets the school ID. + * + * @return the school ID */ - public String getUrn() { - return urn; + public String getSchoolId() { + return schoolId; } /** - * Sets the urn. - * - * @param urn - * the urn to set + * Sets the school ID. + * + * @param schoolId + * the school ID to set */ - public void setUrn(final String urn) { - this.urn = urn; + public void setSchoolId(final String schoolId) { + this.schoolId = schoolId; } /** - * Gets the name. - * - * @return the name + * Gets the country code. + * + * @return the country code */ - public String getName() { - return name; + public String getCountryCode() { + return countryCode; } /** - * Sets the name. - * - * @param name - * the name to set + * Sets the country code. + * + * @param countryCode + * the country code to set */ - public void setName(final String name) { - this.name = name; + public void setCountryCode(final String countryCode) { + this.countryCode = countryCode; } /** - * Gets the postcode. - * - * @return the postcode + * Gets the school name. + * + * @return the school name */ - public String getPostcode() { - return postcode; + public String getSchoolName() { + return schoolName; } /** - * Sets the postcode. - * - * @param postcode - * the postcode to set + * Sets the school name. + * + * @param schoolName + * the school name to set */ - public void setPostcode(final String postcode) { - this.postcode = postcode; + public void setSchoolName(final String schoolName) { + this.schoolName = schoolName; } + /** + * Gets the town. + * + * @return the town + */ + public String getTown() { + return town; + } + + /** + * Sets the town. + * + * @param town + * the town to set + */ + public void setTown(final String town) { + this.town = town; + } + + /** + * Gets the postal code. + * + * @return the postal code + */ + public String getPostalCode() { + return postalCode; + } + + /** + * Sets the postal code. + * + * @param postalCode + * the postal code to set + */ + public void setPostalCode(final String postalCode) { + this.postalCode = postalCode; + } + + /** + * Gets the excluded status. + * + * @return whether the school is excluded when searching for schools by name + */ + public Boolean getExcluded() { + return excluded; + } + + /** + * Sets the excluded status. + * + * @param excluded + * whether the school should be excluded when searching for schools by name + */ + public void setExcluded(final Boolean excluded) { + this.excluded = excluded; + } /** * Gets the closed status. * * @return whether the school is closed */ - public Boolean isClosed() { + public Boolean getClosed() { return closed; } @@ -146,19 +216,19 @@ public void setClosed(final Boolean closed) { } /** - * Gets the verifiedSchool. + * Gets the data source for the school information. * - * @return the verifiedSchool + * @return the data source */ public SchoolDataSource getDataSource() { return dataSource; } /** - * Sets the dataSource. + * Sets the data source for the school information. * * @param dataSource - * the dataSource to set + * the data source to set */ public void setDataSource(final SchoolDataSource dataSource) { this.dataSource = dataSource; @@ -168,7 +238,7 @@ public void setDataSource(final SchoolDataSource dataSource) { public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((urn == null) ? 0 : urn.hashCode()); + result = prime * result + ((schoolId == null) ? 0 : schoolId.hashCode()); return result; } @@ -184,11 +254,11 @@ public boolean equals(final Object obj) { return false; } School other = (School) obj; - if (urn == null) { - if (other.urn != null) { + if (schoolId == null) { + if (other.schoolId != null) { return false; } - } else if (!urn.equals(other.urn)) { + } else if (!schoolId.equals(other.schoolId)) { return false; } return true; @@ -196,6 +266,6 @@ public boolean equals(final Object obj) { @Override public String toString() { - return "School [urn=" + urn + ", name=" + name + ", postcode=" + postcode + "]"; + return "School [id=" + schoolId + ", name=" + schoolName + ", postcode=" + postalCode + "]"; } } diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java index f3579742ad..6c2afab0f3 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java @@ -86,11 +86,14 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -794,8 +797,8 @@ public Response getContentProblems(@Context final HttpServletRequest request, * - if searching by role * @param schoolOther * - if searching by school other field. - * @param schoolURN - * - if searching by school by the URN. + * @param schoolId + * - if searching for school by ID. * @param emailVerificationStatus * - if searching by email verification status * @return a userDTO or a segue error response @@ -808,7 +811,7 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, @QueryParam("id") final Long userId, @QueryParam("email") @Nullable final String email, @QueryParam("familyName") @Nullable final String familyName, @QueryParam("role") @Nullable final Role role, @QueryParam("schoolOther") @Nullable final String schoolOther, - @QueryParam("schoolURN") @Nullable final String schoolURN, + @QueryParam("schoolId") @Nullable final String schoolId, @QueryParam("emailVerificationStatus") @Nullable final EmailVerificationStatus emailVerificationStatus) { RegisteredUserDTO currentUser; @@ -825,7 +828,7 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, && (null == familyName || familyName.isEmpty()) && (null == schoolOther || schoolOther.isEmpty()) && (null == email || email.isEmpty()) - && (null == schoolURN || schoolURN.isEmpty())) { + && (null == schoolId || schoolId.isEmpty())) { return new SegueErrorResponse(Status.FORBIDDEN, "You do not have permission to do wildcard searches.") .toResponse(); @@ -870,8 +873,8 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, userPrototype.setSchoolOther(schoolOther); } - if (null != schoolURN) { - userPrototype.setSchoolId(schoolURN); + if (null != schoolId) { + userPrototype.setSchoolId(schoolId); } if (null != emailVerificationStatus) { diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java index 05139f4642..65bf91d3c9 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java @@ -247,7 +247,7 @@ public enum BooleanOperator { AND, OR, NOT } - public static final String SCHOOLS_INDEX_BASE = "schools"; + public static final String SCHOOLS_INDEX_BASE = "schools_2026"; public enum SCHOOLS_INDEX_TYPE { METADATA("metadata"), SCHOOL_SEARCH("school"); @@ -443,6 +443,7 @@ public enum SegueServerLogType implements LogType { public static final String DEFAULT_SEARCH_RESULT_LIMIT_AS_STRING = "25"; public static final Integer MAX_SEARCH_RESULT_LIMIT = 350; + public static final Integer MAX_SCHOOLS_RESULT_LIMIT = 50; public static final Integer SEARCH_TEXT_CHAR_LIMIT = 1000; public static final Integer CONTACT_FORM_CHAR_LIMIT = 5000; @@ -512,18 +513,15 @@ public enum EventFilterOption { public static final String ID_SEPARATOR = "|"; public static final String ESCAPED_ID_SEPARATOR = "\\" + ID_SEPARATOR; - // School List loading - raw data - public static final String SCHOOL_URN_FIELDNAME = "URN"; - public static final String SCHOOL_ESTABLISHMENT_NAME_FIELDNAME = "EstablishmentName"; - public static final String SCHOOL_POSTCODE_FIELDNAME = "Postcode"; - public static final String SCHOOL_DATA_SOURCE_FIELDNAME = "DataSource"; - public static final String SCHOOL_CLOSED_FIELDNAME = "Closed"; - - // School List loading POJO fields - public static final String SCHOOL_URN_FIELDNAME_POJO = "urn"; - public static final String SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO = "name"; - public static final String SCHOOL_POSTCODE_FIELDNAME_POJO = "postcode"; - public static final String SCHOOL_CLOSED_FIELDNAME_POJO = "closed"; + // School List loading + public static final String SCHOOL_ID_FIELDNAME = "schoolId"; + public static final String SCHOOL_COUNTRY_CODE_FIELDNAME = "countryCode"; + public static final String SCHOOL_NAME_FIELDNAME = "schoolName"; + public static final String SCHOOL_TOWN_FIELDNAME = "town"; + public static final String SCHOOL_POSTCODE_FIELDNAME = "postalCode"; + public static final String SCHOOL_EXCLUDED_FIELDNAME = "excluded"; + public static final String SCHOOL_CLOSED_FIELDNAME = "closed"; + public static final String SCHOOL_DATA_SOURCE_FIELDNAME = "dataSource"; // User School Reporting diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java index 30167c14e7..518ac0c251 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java @@ -40,9 +40,11 @@ import jakarta.ws.rs.core.Response.ResponseBuilder; import jakarta.ws.rs.core.Response.Status; import java.io.IOException; -import java.util.Arrays; +import java.util.Collections; import java.util.List; +import static uk.ac.cam.cl.dtg.segue.api.Constants.*; + /** * Segue School Lookup service. * @@ -70,8 +72,8 @@ public SchoolLookupServiceFacade(final SchoolListReader schoolListReader) { * * @param request * - for caching purposes. - * @param schoolURN - * - find by urn. + * @param schoolId + * - find by school ID. * @param searchQuery * - query to search fields against. * @param limit @@ -84,11 +86,21 @@ public SchoolLookupServiceFacade(final SchoolListReader schoolListReader) { @GZIP @Operation(summary = "List all schools matching provided criteria.") public Response schoolSearch(@Context final Request request, @QueryParam("query") final String searchQuery, - @QueryParam("urn") final String schoolURN, @QueryParam("limit") final Integer limit) { + @QueryParam("countryCode") final String countryCode, @QueryParam("schoolId") final String schoolId, + @QueryParam("limit") final Integer limit) { + + if (null != limit && limit > MAX_SCHOOLS_RESULT_LIMIT) { + return SegueErrorResponse.getBadRequestResponse("Too many schools requested!"); + } - if ((null == searchQuery || searchQuery.isEmpty()) && (null == schoolURN || schoolURN.isEmpty())) { - return new SegueErrorResponse(Status.BAD_REQUEST, "You must provide a search query or school URN") - .toResponse(); + boolean queryEmpty = searchQuery == null || searchQuery.isEmpty(); + boolean schoolIdEmpty = schoolId == null || schoolId.isEmpty(); + boolean countryCodeEmpty = countryCode == null || countryCode.isEmpty(); + if ((queryEmpty && schoolIdEmpty) || (!queryEmpty && !schoolIdEmpty)) { + return SegueErrorResponse.getBadRequestResponse("You must provide either a search query, or a school ID."); + } + if (!schoolIdEmpty && !countryCodeEmpty) { + return SegueErrorResponse.getBadRequestResponse("You cannot filter by country when searching by school ID."); } EntityTag etag = new EntityTag(schoolListReader.getDataLastModifiedDate()); @@ -109,10 +121,10 @@ public Response schoolSearch(@Context final Request request, @QueryParam("query" List list; try { - if (schoolURN != null && !schoolURN.isEmpty()) { - list = Arrays.asList(schoolListReader.findSchoolById(schoolURN)); + if (schoolId != null && !schoolId.isEmpty()) { + list = Collections.singletonList(schoolListReader.findSchoolById(schoolId)); } else { - list = schoolListReader.findSchoolByNameOrPostCode(searchQuery, limit); + list = schoolListReader.findSchoolByNameOrPostCode(searchQuery, countryCode, limit); } } catch (UnableToIndexSchoolsException | SegueSearchException | IOException e) { @@ -120,7 +132,7 @@ public Response schoolSearch(@Context final Request request, @QueryParam("query" log.error(message, e); return new SegueErrorResponse(Status.INTERNAL_SERVER_ERROR, message, e).toResponse(); } catch (NumberFormatException e) { - return new SegueErrorResponse(Status.BAD_REQUEST, "The school urn provided is invalid.").toResponse(); + return new SegueErrorResponse(Status.BAD_REQUEST, "The school ID provided is invalid.").toResponse(); } return Response.ok(list).tag(etag).cacheControl(cc).build(); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java index 66e370d045..32f57e8f3a 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java @@ -730,7 +730,7 @@ public Response getUserIdToSchoolMap(@Context final HttpServletRequest httpServl if (null != school) { builder.put(user.getId().toString(), school); } else { - // The school once existed in the list but no longer does. Set the name to be the URN: + // The school once existed in the list but no longer does. Set the name to be the ID: builder.put(user.getId().toString(), ImmutableMap.of("name", user.getSchoolId())); } } else if (user.getSchoolOther() != null && !user.getSchoolOther().isEmpty()) { diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 522b931eea..5cef98a961 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -76,17 +76,20 @@ public SchoolListReader(final ISearchProvider searchProvider) { } /** - * findSchoolByNameOrPostCode. + * Search for schools by ID, name or postcode. Filterable by country. Excludes schools marked as closed or excluded. * * @param searchQuery - * - school to search for - either name or postcode. + * - school to search for - either ID, name or postcode. + * @param countryCode + * - country code to filter by. * @param limit * - the number of results to return. * @return list of schools matching the criteria or an empty list. * @throws UnableToIndexSchoolsException * - if there is an error access the index of schools. */ - public List findSchoolByNameOrPostCode(final String searchQuery, @Nullable final Integer limit) throws UnableToIndexSchoolsException, SegueSearchException { + public List findSchoolByNameOrPostCode(final String searchQuery, @Nullable final String countryCode, + @Nullable final Integer limit) throws UnableToIndexSchoolsException, SegueSearchException { if (!this.ensureSchoolList()) { log.error("Unable to ensure school search cache."); throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated"); @@ -94,19 +97,22 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab Integer queryLimit = limit == null ? DEFAULT_RESULTS_LIMIT : limit; - BooleanInstruction schoolSearchInstruction = new BooleanInstruction(); - schoolSearchInstruction.must(new MatchInstruction(SCHOOL_CLOSED_FIELDNAME_POJO, "false")); - - // At least one of these fields must match, with some fuzziness - BooleanInstruction textSearchInstruction = new BooleanInstruction(); - textSearchInstruction.should(new MatchInstruction(SCHOOL_URN_FIELDNAME_POJO, searchQuery, 2L, true)); - textSearchInstruction.should(new MatchInstruction(SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO, searchQuery, 2L, true)); - textSearchInstruction.should(new MatchInstruction(SCHOOL_POSTCODE_FIELDNAME_POJO, searchQuery, 2L, true)); - schoolSearchInstruction.must(textSearchInstruction); + BooleanInstruction matchInstruction = new BooleanInstruction(); + // Exclude excluded/closed schools + matchInstruction.must(new MatchInstruction(SCHOOL_EXCLUDED_FIELDNAME, "false", null, false)); + matchInstruction.must(new MatchInstruction(SCHOOL_CLOSED_FIELDNAME, "false", null, false)); + // Compulsorily match country code, if provided. Needs to be a raw field since GB- country codes contain hyphens. + if (null != countryCode && !countryCode.isEmpty()) { + matchInstruction.must(new MatchInstruction(SCHOOL_COUNTRY_CODE_FIELDNAME + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, + countryCode, null, false)); + } + // Attempt to match on school ID, name & postcode + matchInstruction.should(new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, false)); + matchInstruction.should(new MatchInstruction(SCHOOL_NAME_FIELDNAME, searchQuery, null, true)); + matchInstruction.should(new MatchInstruction(SCHOOL_POSTCODE_FIELDNAME, searchQuery, null, true)); List schoolSearchResults = searchProvider.nestedMatchSearch(SCHOOLS_INDEX_BASE, - SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), 0, queryLimit, schoolSearchInstruction, null, null - ).getResults(); + SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), 0, queryLimit, matchInstruction, null, null).getResults(); List resultList = Lists.newArrayList(); for (String schoolString : schoolSearchResults) { @@ -124,7 +130,7 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab /** * Find school by Id. * - * @param schoolURN + * @param schoolId * - to search for. * @return school. * @throws UnableToIndexSchoolsException @@ -136,7 +142,7 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab * @throws JsonParseException * - if the school data is malformed */ - public School findSchoolById(final String schoolURN) throws UnableToIndexSchoolsException, JsonParseException, + public School findSchoolById(final String schoolId) throws UnableToIndexSchoolsException, JsonParseException, JsonMappingException, IOException, SegueSearchException { if (!this.ensureSchoolList()) { @@ -144,7 +150,7 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated"); } - MatchInstruction searchInstruction = new MatchInstruction(SCHOOL_URN_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, schoolURN); + MatchInstruction searchInstruction = new MatchInstruction(SCHOOL_ID_FIELDNAME, schoolId); List matchingSchoolList = searchProvider.nestedMatchSearch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), 0, DEFAULT_RESULTS_LIMIT, searchInstruction, null, null @@ -155,7 +161,7 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools } if (matchingSchoolList.size() > 1) { - log.error("Error while looking up school up by id! More than one match for '{}' results: {}", schoolURN, matchingSchoolList); + log.error("Error while looking up school up by id! More than one match for '{}' results: {}", schoolId, matchingSchoolList); } return mapper.readValue(matchingSchoolList.getFirst(), School.class); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java index c0c39f4168..8281b3f9c1 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java @@ -27,7 +27,6 @@ import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import uk.ac.cam.cl.dtg.segue.api.Constants; import uk.ac.cam.cl.dtg.segue.search.ElasticSearchProvider; import uk.ac.cam.cl.dtg.segue.search.SegueSearchException; @@ -40,6 +39,8 @@ import java.util.Map; import java.util.function.Function; +import static uk.ac.cam.cl.dtg.segue.api.Constants.*; + /** * Created by Ian on 17/10/2016. */ @@ -60,9 +61,9 @@ public class ElasticSearchIndexer extends ElasticSearchProvider { @Inject public ElasticSearchIndexer(final ElasticsearchClient searchClient) { super(searchClient); - rawFieldsListByType.put("content", Lists.newArrayList("id", "title", "subtitle")); - rawFieldsListByType.put("school", Lists.newArrayList("urn")); - nestedFieldsByType.put("content", Lists.newArrayList("audience")); + rawFieldsListByType.put(CONTENT_INDEX_TYPE.CONTENT.toString(), Lists.newArrayList("id", "title", "subtitle")); + rawFieldsListByType.put(SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), Lists.newArrayList("countryCode")); + nestedFieldsByType.put(CONTENT_INDEX_TYPE.CONTENT.toString(), Lists.newArrayList("audience")); } @@ -319,11 +320,11 @@ private void sendMappingCorrections(final String typedIndex, final String indexT // Add mapping to specify raw, un-analyzed fields for (String fieldName : this.rawFieldsListByType.getOrDefault(indexType, Collections.emptyList())) { - log.debug("Sending raw mapping correction for {}." + Constants.UNPROCESSED_SEARCH_FIELD_SUFFIX, fieldName); + log.debug("Sending raw mapping correction for {}." + UNPROCESSED_SEARCH_FIELD_SUFFIX, fieldName); properties.put(fieldName, Property.of(p -> p .text(t -> t .fields( - Constants.UNPROCESSED_SEARCH_FIELD_SUFFIX, + UNPROCESSED_SEARCH_FIELD_SUFFIX, Property.of(k -> k.keyword(kf -> kf)) ) ) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java index d21c648ea1..8fc0300fdd 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java @@ -34,11 +34,11 @@ */ class SchoolIndexer { private static final Logger log = LoggerFactory.getLogger(SchoolIndexer.class); - private ElasticSearchIndexer es; - private ContentSubclassMapper mapper; - private String schoolsListPath; + private final ElasticSearchIndexer es; + private final ContentSubclassMapper mapper; + private final String schoolsListPath; - SchoolIndexer(ElasticSearchIndexer es, ContentSubclassMapper mapper, String schoolsListPath) { + SchoolIndexer(final ElasticSearchIndexer es, final ContentSubclassMapper mapper, final String schoolsListPath) { this.es = es; this.mapper = mapper; this.schoolsListPath = schoolsListPath; @@ -65,8 +65,8 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx for (School school : schoolList) { try { - indexList.add(immutableEntry(school.getUrn(), objectMapper.writeValueAsString(school))); - } catch (JsonProcessingException e) { + indexList.add(immutableEntry(school.getSchoolId(), objectMapper.writeValueAsString(school))); + } catch (final JsonProcessingException e) { log.error("Unable to serialize the school object into json.", e); } } @@ -75,23 +75,23 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx try { es.indexObject(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.METADATA.toString(), objectMapper.writeValueAsString( ImmutableMap.of("lastModified", f.lastModified())), "sourceFile"); - } catch (SegueSearchException e) { + } catch (final SegueSearchException e) { log.error("Unable to index school list metadata.", e); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { log.error("Unable to serialise school list last modified date to JSON.", e); } try { es.bulkIndexWithIDs(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), indexList); log.info("School list index request complete."); - } catch (SegueSearchException e) { + } catch (final SegueSearchException e) { log.error("Unable to complete bulk index operation for schools list.", e); } // Create an alias (could be anything) to prevent this schools index from being garbage-collected by ElasticSearchIndexer.expungeOldIndices List allSchoolTypes = Arrays.stream(SCHOOLS_INDEX_TYPE.values()) .map(SCHOOLS_INDEX_TYPE::toString).collect(Collectors.toList()); - es.addOrMoveIndexAlias("schools-latest", SCHOOLS_INDEX_BASE, allSchoolTypes); + es.addOrMoveIndexAlias(String.format("%s_latest", SCHOOLS_INDEX_BASE), SCHOOLS_INDEX_BASE, allSchoolTypes); } /** @@ -121,36 +121,35 @@ private synchronized List loadAndBuildSchoolList() throws UnableToIndexS } // We expect the columns to have the following names/structure and be UTF-8 encoded: - // URN | EstablishmentName | Postcode | Closed | DataSource + // schoolId | countryCode | schoolName | town | postalCode | excluded | closed | dataSource String[] schoolArray; while ((schoolArray = reader.readNext()) != null) { try { School.SchoolDataSource source = School.SchoolDataSource .valueOf(schoolArray[fieldNameMapping.get(Constants.SCHOOL_DATA_SOURCE_FIELDNAME)]); - School schoolToSave = new School(schoolArray[fieldNameMapping.get(Constants.SCHOOL_URN_FIELDNAME)], - schoolArray[fieldNameMapping.get(Constants.SCHOOL_ESTABLISHMENT_NAME_FIELDNAME)], + School schoolToSave = new School(schoolArray[fieldNameMapping.get(Constants.SCHOOL_ID_FIELDNAME)], + schoolArray[fieldNameMapping.get(Constants.SCHOOL_COUNTRY_CODE_FIELDNAME)], + schoolArray[fieldNameMapping.get(Constants.SCHOOL_NAME_FIELDNAME)], + schoolArray[fieldNameMapping.get(Constants.SCHOOL_TOWN_FIELDNAME)], schoolArray[fieldNameMapping.get(Constants.SCHOOL_POSTCODE_FIELDNAME)], - // CSV file contains string "t" and "f" values to denote true and false, but need a boolean: - "t".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]), + // CSV file contains string "true" and "false" values to denote true and false, but need a boolean: + "true".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_EXCLUDED_FIELDNAME)]), + "true".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]), source); - if (null == schoolToSave.getPostcode() || schoolToSave.getPostcode().isEmpty()) { - log.warn("School with missing postcode! URN: {}", schoolToSave.getUrn()); - } - schools.add(schoolToSave); - } catch (IndexOutOfBoundsException e) { + } catch (final IndexOutOfBoundsException e) { // This happens when the school does not have the required data log.warn("Unable to load school into list due to missing required fields: {}", Arrays.toString(schoolArray)); } } - } catch (FileNotFoundException e) { + } catch (final FileNotFoundException e) { log.error("Unable to locate the file requested", e); throw new UnableToIndexSchoolsException("Unable to locate the file requested", e); - } catch (IOException e) { + } catch (final IOException e) { throw new UnableToIndexSchoolsException("Unable to load the file requested", e); - } catch (CsvValidationException e) { + } catch (final CsvValidationException e) { throw new UnableToIndexSchoolsException("Unable to parse the file requested", e); }