From 833c090d86ecdda3f1db1eea2325229e1dbf7a66 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 29 Apr 2026 12:05:56 +0100 Subject: [PATCH 01/19] Add new schools properties --- .../ac/cam/cl/dtg/isaac/dos/users/School.java | 166 +++++++++++++++++- 1 file changed, 161 insertions(+), 5 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java index 7190885c96..3868ecc67e 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java @@ -20,9 +20,17 @@ * */ public class School { + // Old fieldnames, to be removed private String urn; private String name; private String postcode; + + private String schoolId; + private String countryCode; + private String schoolName; + private String town; + private String postalCode; + private Boolean excluded; private Boolean closed; /** @@ -68,6 +76,39 @@ public School(final String urn, final String name, final String postcode, final this.closed = closed; } + /** + * Full constructor. + * + * @param schoolId + * - unique school ID + * @param countryCode + * - country code for the school + * @param schoolName + * - name of the school + * @param town + * - name of the town where the school is located + * @param postalCode + * - postal code of the school + * @param excluded + * - whether the school is excluded when searching for schools by name + * @param closed + * - whether the school is closed + * @param dataSource + * - data source of this information + */ + public School(final String schoolId, final String countryCode, final String schoolName, final String town, + final String postalCode, final Boolean excluded, final Boolean closed, final SchoolDataSource dataSource) { + this.schoolId = schoolId; + this.countryCode = countryCode; + this.schoolName = schoolName; + this.town = town; + this.postalCode = postalCode; + this.excluded = excluded; + this.closed = closed; + this.dataSource = dataSource; + } + + /** * Gets the urn. * @@ -126,12 +167,127 @@ public void setPostcode(final String postcode) { } + /** + * Gets the school id. + * + * @return the school id + */ + public String getSchoolId() { + return schoolId; + } + + /** + * Sets the school id. + * + * @param schoolId + * the school id to set + */ + public void setSchoolId(final String schoolId) { + this.schoolId = schoolId; + } + + + /** + * Gets the country code. + * + * @return the country code + */ + public String getCountryCode() { + return countryCode; + } + + /** + * Sets the country code. + * + * @param countryCode + * the country code to set + */ + public void setCountryCode(final String countryCode) { + this.countryCode = countryCode; + } + + /** + * Gets the school name. + * + * @return the school name + */ + public String getSchoolName() { + return schoolName; + } + + /** + * Sets the school name. + * + * @param schoolName + * the school name to set + */ + public void setSchoolName(final String schoolName) { + this.schoolName = schoolName; + } + + /** + * Gets the town. + * + * @return the town + */ + public String getTown() { + return town; + } + + /** + * Sets the town. + * + * @param town + * the town to set + */ + public void setTown(final String town) { + this.town = town; + } + + /** + * Gets the postal code. + * + * @return the postal code + */ + public String getPostalCode() { + return postalCode; + } + + /** + * Sets the postal code. + * + * @param postalCode + * the postal code to set + */ + public void setPostalCode(final String postalCode) { + this.postalCode = postalCode; + } + + /** + * Gets the excluded status. + * + * @return whether the school is excluded when searching for schools by name + */ + public Boolean getExcluded() { + return excluded; + } + + /** + * Sets the excluded status. + * + * @param excluded + * whether the school should be excluded when searching for schools by name + */ + public void setExcluded(final Boolean excluded) { + this.excluded = excluded; + } + /** * Gets the closed status. * * @return whether the school is closed */ - public Boolean isClosed() { + public Boolean getClosed() { return closed; } @@ -146,19 +302,19 @@ public void setClosed(final Boolean closed) { } /** - * Gets the verifiedSchool. + * Gets the data source for the school information. * - * @return the verifiedSchool + * @return the data source */ public SchoolDataSource getDataSource() { return dataSource; } /** - * Sets the dataSource. + * Sets the data source for the school information. * * @param dataSource - * the dataSource to set + * the data source to set */ public void setDataSource(final SchoolDataSource dataSource) { this.dataSource = dataSource; From 9d3d585a115daa648ebe8e77cb44e8c88b506158 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 29 Apr 2026 12:09:02 +0100 Subject: [PATCH 02/19] Ingest schools list using new properties --- .../uk/ac/cam/cl/dtg/isaac/dos/users/School.java | 2 +- .../java/uk/ac/cam/cl/dtg/segue/api/Constants.java | 13 ++++++++----- .../uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java | 13 ++++++++----- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java index 3868ecc67e..4f1c2860bd 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java @@ -37,7 +37,7 @@ public class School { * Enum to represent where this school object was created. */ public enum SchoolDataSource { - GOVERNMENT_UK, GOVERNMENT_IE, GOVERNMENT_SCO, GOVERNMENT_WAL, GOVERNMENT_NI, USER_ENTERED; + GOVERNMENT_UK, GOVERNMENT_IE, GOVERNMENT_SCT, GOVERNMENT_SCT_IND, GOVERNMENT_WLS, GOVERNMENT_NIR, USER_ENTERED; @Override public String toString() { diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java index 5a6e27249b..468eb74ffb 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java @@ -510,11 +510,14 @@ public enum EventFilterOption { public static final String ESCAPED_ID_SEPARATOR = "\\" + ID_SEPARATOR; // School List loading - raw data - public static final String SCHOOL_URN_FIELDNAME = "URN"; - public static final String SCHOOL_ESTABLISHMENT_NAME_FIELDNAME = "EstablishmentName"; - public static final String SCHOOL_POSTCODE_FIELDNAME = "Postcode"; - public static final String SCHOOL_DATA_SOURCE_FIELDNAME = "DataSource"; - public static final String SCHOOL_CLOSED_FIELDNAME = "Closed"; + public static final String SCHOOL_ID_FIELDNAME = "schoolId"; + public static final String SCHOOL_COUNTRY_CODE_FIELDNAME = "countryCode"; + public static final String SCHOOL_NAME_FIELDNAME = "schoolName"; + public static final String SCHOOL_TOWN_FIELDNAME = "town"; + public static final String SCHOOL_POSTCODE_FIELDNAME = "postalCode"; + public static final String SCHOOL_EXCLUDED_FIELDNAME = "excluded"; + public static final String SCHOOL_CLOSED_FIELDNAME = "closed"; + public static final String SCHOOL_DATA_SOURCE_FIELDNAME = "dataSource"; // School List loading POJO fields public static final String SCHOOL_URN_FIELDNAME_POJO = "urn"; diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java index d21c648ea1..cb7a7f4457 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java @@ -121,18 +121,21 @@ private synchronized List loadAndBuildSchoolList() throws UnableToIndexS } // We expect the columns to have the following names/structure and be UTF-8 encoded: - // URN | EstablishmentName | Postcode | Closed | DataSource + // schoolId | countryCode | schoolName | town | postalCode | excluded | closed | dataSource String[] schoolArray; while ((schoolArray = reader.readNext()) != null) { try { School.SchoolDataSource source = School.SchoolDataSource .valueOf(schoolArray[fieldNameMapping.get(Constants.SCHOOL_DATA_SOURCE_FIELDNAME)]); - School schoolToSave = new School(schoolArray[fieldNameMapping.get(Constants.SCHOOL_URN_FIELDNAME)], - schoolArray[fieldNameMapping.get(Constants.SCHOOL_ESTABLISHMENT_NAME_FIELDNAME)], + School schoolToSave = new School(schoolArray[fieldNameMapping.get(Constants.SCHOOL_ID_FIELDNAME)], + schoolArray[fieldNameMapping.get(Constants.SCHOOL_COUNTRY_CODE_FIELDNAME)], + schoolArray[fieldNameMapping.get(Constants.SCHOOL_NAME_FIELDNAME)], + schoolArray[fieldNameMapping.get(Constants.SCHOOL_TOWN_FIELDNAME)], schoolArray[fieldNameMapping.get(Constants.SCHOOL_POSTCODE_FIELDNAME)], - // CSV file contains string "t" and "f" values to denote true and false, but need a boolean: - "t".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]), + // CSV file contains string "true" and "false" values to denote true and false, but need a boolean: + "true".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_EXCLUDED_FIELDNAME)]), + "true".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]), source); if (null == schoolToSave.getPostcode() || schoolToSave.getPostcode().isEmpty()) { From e94b756421e8c97ed2dfcc621dc0a69c7de08250 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 29 Apr 2026 13:31:59 +0100 Subject: [PATCH 03/19] Replace & remove old schools properties --- .../ac/cam/cl/dtg/isaac/api/EventsFacade.java | 2 +- .../ac/cam/cl/dtg/isaac/dos/users/School.java | 105 ++---------------- .../ac/cam/cl/dtg/segue/api/AdminFacade.java | 14 +-- .../uk/ac/cam/cl/dtg/segue/api/Constants.java | 8 +- .../segue/api/SchoolLookupServiceFacade.java | 18 +-- .../ac/cam/cl/dtg/segue/api/UsersFacade.java | 2 +- .../segue/dao/schools/SchoolListReader.java | 14 +-- .../cam/cl/dtg/segue/etl/SchoolIndexer.java | 6 +- 8 files changed, 38 insertions(+), 131 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java index 7a89602d32..5702775924 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/api/EventsFacade.java @@ -566,7 +566,7 @@ eventId, new Date(), currentUser.getGivenName(), if (schoolId != null) { School school = schoolListReader.findSchoolById(schoolId); if (null != school) { - resultRow.add(school.getName()); + resultRow.add(school.getSchoolName()); } else { resultRow.add(schoolId); } diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java index 4f1c2860bd..4fd1a0bf9e 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java @@ -20,11 +20,6 @@ * */ public class School { - // Old fieldnames, to be removed - private String urn; - private String name; - private String postcode; - private String schoolId; private String countryCode; private String schoolName; @@ -54,28 +49,6 @@ public School() { } - /** - * Full constructor. - * - * @param urn - * - unique id - * @param name - * - name of the school. - * @param postcode - * -postcode of the school - * @param dataSource - * -dataSource of this information - * @param closed - * - whether the school is closed - */ - public School(final String urn, final String name, final String postcode, final Boolean closed, final SchoolDataSource dataSource) { - this.urn = urn; - this.name = name; - this.postcode = postcode; - this.dataSource = dataSource; - this.closed = closed; - } - /** * Full constructor. * @@ -108,85 +81,25 @@ public School(final String schoolId, final String countryCode, final String scho this.dataSource = dataSource; } - - /** - * Gets the urn. - * - * @return the urn - */ - public String getUrn() { - return urn; - } - - /** - * Sets the urn. - * - * @param urn - * the urn to set - */ - public void setUrn(final String urn) { - this.urn = urn; - } - /** - * Gets the name. - * - * @return the name - */ - public String getName() { - return name; - } - - /** - * Sets the name. - * - * @param name - * the name to set - */ - public void setName(final String name) { - this.name = name; - } - - /** - * Gets the postcode. - * - * @return the postcode - */ - public String getPostcode() { - return postcode; - } - - /** - * Sets the postcode. - * - * @param postcode - * the postcode to set - */ - public void setPostcode(final String postcode) { - this.postcode = postcode; - } - - - /** - * Gets the school id. + * Gets the school ID. * - * @return the school id + * @return the school ID */ public String getSchoolId() { return schoolId; } /** - * Sets the school id. + * Sets the school ID. * * @param schoolId - * the school id to set + * the school ID to set */ public void setSchoolId(final String schoolId) { this.schoolId = schoolId; } - /** * Gets the country code. * @@ -324,7 +237,7 @@ public void setDataSource(final SchoolDataSource dataSource) { public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((urn == null) ? 0 : urn.hashCode()); + result = prime * result + ((schoolId == null) ? 0 : schoolId.hashCode()); return result; } @@ -340,11 +253,11 @@ public boolean equals(final Object obj) { return false; } School other = (School) obj; - if (urn == null) { - if (other.urn != null) { + if (schoolId == null) { + if (other.schoolId != null) { return false; } - } else if (!urn.equals(other.urn)) { + } else if (!schoolId.equals(other.schoolId)) { return false; } return true; @@ -352,6 +265,6 @@ public boolean equals(final Object obj) { @Override public String toString() { - return "School [urn=" + urn + ", name=" + name + ", postcode=" + postcode + "]"; + return "School [urn=" + schoolId + ", name=" + schoolName + ", postcode=" + postalCode + "]"; } } diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java index 9563df2052..8074a5e038 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java @@ -806,8 +806,8 @@ public Response getContentProblems(@Context final HttpServletRequest request, * - if searching by school other field. * @param postcode * - if searching by postcode. - * @param schoolURN - * - if searching by school by the URN. + * @param schoolId + * - if searching for school by ID. * @param emailVerificationStatus * - if searching by email verification status * @return a userDTO or a segue error response @@ -822,7 +822,7 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, @QueryParam("schoolOther") @Nullable final String schoolOther, @QueryParam("postcode") @Nullable final String postcode, @QueryParam("postcodeRadius") @Nullable final String postcodeRadius, - @QueryParam("schoolURN") @Nullable final String schoolURN, + @QueryParam("schoolId") @Nullable final String schoolId, @QueryParam("emailVerificationStatus") @Nullable final EmailVerificationStatus emailVerificationStatus) { RegisteredUserDTO currentUser; @@ -839,7 +839,7 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, && (null == familyName || familyName.isEmpty()) && (null == schoolOther || schoolOther.isEmpty()) && (null == email || email.isEmpty()) - && (null == schoolURN || schoolURN.isEmpty()) + && (null == schoolId || schoolId.isEmpty()) && (null == postcode || postcode.isEmpty())) { return new SegueErrorResponse(Status.FORBIDDEN, "You do not have permission to do wildcard searches.") .toResponse(); @@ -885,8 +885,8 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, userPrototype.setSchoolOther(schoolOther); } - if (null != schoolURN) { - userPrototype.setSchoolId(schoolURN); + if (null != schoolId) { + userPrototype.setSchoolId(schoolId); } if (null != emailVerificationStatus) { @@ -915,7 +915,7 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, if (userDTO.getSchoolId() != null) { School school = this.schoolReader.findSchoolById(userDTO.getSchoolId()); if (school != null) { - String schoolPostCode = school.getPostcode(); + String schoolPostCode = school.getPostalCode(); if (null == schoolPostCode || schoolPostCode.isEmpty()) { continue; } diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java index 468eb74ffb..8e746e11a7 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java @@ -509,7 +509,7 @@ public enum EventFilterOption { public static final String ID_SEPARATOR = "|"; public static final String ESCAPED_ID_SEPARATOR = "\\" + ID_SEPARATOR; - // School List loading - raw data + // School List loading public static final String SCHOOL_ID_FIELDNAME = "schoolId"; public static final String SCHOOL_COUNTRY_CODE_FIELDNAME = "countryCode"; public static final String SCHOOL_NAME_FIELDNAME = "schoolName"; @@ -519,12 +519,6 @@ public enum EventFilterOption { public static final String SCHOOL_CLOSED_FIELDNAME = "closed"; public static final String SCHOOL_DATA_SOURCE_FIELDNAME = "dataSource"; - // School List loading POJO fields - public static final String SCHOOL_URN_FIELDNAME_POJO = "urn"; - public static final String SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO = "name"; - public static final String SCHOOL_POSTCODE_FIELDNAME_POJO = "postcode"; - public static final String SCHOOL_CLOSED_FIELDNAME_POJO = "closed"; - // User School Reporting /** diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java index 30167c14e7..55061c82e7 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java @@ -40,7 +40,7 @@ import jakarta.ws.rs.core.Response.ResponseBuilder; import jakarta.ws.rs.core.Response.Status; import java.io.IOException; -import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -70,8 +70,8 @@ public SchoolLookupServiceFacade(final SchoolListReader schoolListReader) { * * @param request * - for caching purposes. - * @param schoolURN - * - find by urn. + * @param schoolId + * - find by school ID. * @param searchQuery * - query to search fields against. * @param limit @@ -84,10 +84,10 @@ public SchoolLookupServiceFacade(final SchoolListReader schoolListReader) { @GZIP @Operation(summary = "List all schools matching provided criteria.") public Response schoolSearch(@Context final Request request, @QueryParam("query") final String searchQuery, - @QueryParam("urn") final String schoolURN, @QueryParam("limit") final Integer limit) { + @QueryParam("schoolId") final String schoolId, @QueryParam("limit") final Integer limit) { - if ((null == searchQuery || searchQuery.isEmpty()) && (null == schoolURN || schoolURN.isEmpty())) { - return new SegueErrorResponse(Status.BAD_REQUEST, "You must provide a search query or school URN") + if ((null == searchQuery || searchQuery.isEmpty()) && (null == schoolId || schoolId.isEmpty())) { + return new SegueErrorResponse(Status.BAD_REQUEST, "You must provide a search query or school ID") .toResponse(); } @@ -109,8 +109,8 @@ public Response schoolSearch(@Context final Request request, @QueryParam("query" List list; try { - if (schoolURN != null && !schoolURN.isEmpty()) { - list = Arrays.asList(schoolListReader.findSchoolById(schoolURN)); + if (schoolId != null && !schoolId.isEmpty()) { + list = Collections.singletonList(schoolListReader.findSchoolById(schoolId)); } else { list = schoolListReader.findSchoolByNameOrPostCode(searchQuery, limit); } @@ -120,7 +120,7 @@ public Response schoolSearch(@Context final Request request, @QueryParam("query" log.error(message, e); return new SegueErrorResponse(Status.INTERNAL_SERVER_ERROR, message, e).toResponse(); } catch (NumberFormatException e) { - return new SegueErrorResponse(Status.BAD_REQUEST, "The school urn provided is invalid.").toResponse(); + return new SegueErrorResponse(Status.BAD_REQUEST, "The school ID provided is invalid.").toResponse(); } return Response.ok(list).tag(etag).cacheControl(cc).build(); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java index 66e370d045..32f57e8f3a 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/UsersFacade.java @@ -730,7 +730,7 @@ public Response getUserIdToSchoolMap(@Context final HttpServletRequest httpServl if (null != school) { builder.put(user.getId().toString(), school); } else { - // The school once existed in the list but no longer does. Set the name to be the URN: + // The school once existed in the list but no longer does. Set the name to be the ID: builder.put(user.getId().toString(), ImmutableMap.of("name", user.getSchoolId())); } } else if (user.getSchoolOther() != null && !user.getSchoolOther().isEmpty()) { diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 4b09db0ae1..519964dccb 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -94,8 +94,8 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab Integer queryLimit = limit == null ? DEFAULT_RESULTS_LIMIT : limit; List schoolSearchResults = searchProvider.fuzzySearch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), - searchQuery, 0, queryLimit, Map.of(SCHOOL_CLOSED_FIELDNAME_POJO, List.of("false")), null, SCHOOL_URN_FIELDNAME_POJO, - SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO, SCHOOL_POSTCODE_FIELDNAME_POJO) + searchQuery, 0, queryLimit, Map.of(SCHOOL_CLOSED_FIELDNAME, List.of("false")), null, SCHOOL_ID_FIELDNAME, + SCHOOL_NAME_FIELDNAME, SCHOOL_POSTCODE_FIELDNAME) .getResults(); List resultList = Lists.newArrayList(); @@ -114,7 +114,7 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab /** * Find school by Id. * - * @param schoolURN + * @param schoolId * - to search for. * @return school. * @throws UnableToIndexSchoolsException @@ -126,7 +126,7 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab * @throws JsonParseException * - if the school data is malformed */ - public School findSchoolById(final String schoolURN) throws UnableToIndexSchoolsException, JsonParseException, + public School findSchoolById(final String schoolId) throws UnableToIndexSchoolsException, JsonParseException, JsonMappingException, IOException, SegueSearchException { if (!this.ensureSchoolList()) { @@ -137,15 +137,15 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools List matchingSchoolList; matchingSchoolList = searchProvider.findByExactMatch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), - SCHOOL_URN_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, - schoolURN, 0, DEFAULT_RESULTS_LIMIT, null).getResults(); + SCHOOL_ID_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, + schoolId, 0, DEFAULT_RESULTS_LIMIT, null).getResults(); if (matchingSchoolList.isEmpty()) { return null; } if (matchingSchoolList.size() > 1) { - log.error("Error while looking up school up by id! More than one match for '{}' results: {}", schoolURN, matchingSchoolList); + log.error("Error while looking up school up by id! More than one match for '{}' results: {}", schoolId, matchingSchoolList); } return mapper.readValue(matchingSchoolList.getFirst(), School.class); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java index cb7a7f4457..4be7060959 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java @@ -65,7 +65,7 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx for (School school : schoolList) { try { - indexList.add(immutableEntry(school.getUrn(), objectMapper.writeValueAsString(school))); + indexList.add(immutableEntry(school.getSchoolId(), objectMapper.writeValueAsString(school))); } catch (JsonProcessingException e) { log.error("Unable to serialize the school object into json.", e); } @@ -138,8 +138,8 @@ private synchronized List loadAndBuildSchoolList() throws UnableToIndexS "true".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]), source); - if (null == schoolToSave.getPostcode() || schoolToSave.getPostcode().isEmpty()) { - log.warn("School with missing postcode! URN: {}", schoolToSave.getUrn()); + if (null == schoolToSave.getPostalCode() || schoolToSave.getPostalCode().isEmpty()) { + log.warn("School with missing postcode! School ID: {}", schoolToSave.getSchoolId()); } schools.add(schoolToSave); From 0f81ca9fe9a66983813c55a0e43f439a8d49e004 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 29 Apr 2026 14:11:05 +0100 Subject: [PATCH 04/19] Use BooleanInstruction search for school lookup --- .../segue/dao/schools/SchoolListReader.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 519964dccb..c08f7eb7f6 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -25,13 +25,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import uk.ac.cam.cl.dtg.isaac.dos.users.School; +import uk.ac.cam.cl.dtg.segue.search.BooleanInstruction; import uk.ac.cam.cl.dtg.segue.search.ISearchProvider; +import uk.ac.cam.cl.dtg.segue.search.MatchInstruction; import uk.ac.cam.cl.dtg.segue.search.SegueSearchException; import jakarta.annotation.Nullable; import java.io.IOException; import java.util.List; -import java.util.Map; import static uk.ac.cam.cl.dtg.segue.api.Constants.*; @@ -93,10 +94,19 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab Integer queryLimit = limit == null ? DEFAULT_RESULTS_LIMIT : limit; - List schoolSearchResults = searchProvider.fuzzySearch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), - searchQuery, 0, queryLimit, Map.of(SCHOOL_CLOSED_FIELDNAME, List.of("false")), null, SCHOOL_ID_FIELDNAME, - SCHOOL_NAME_FIELDNAME, SCHOOL_POSTCODE_FIELDNAME) - .getResults(); + MatchInstruction excludeClosedSchools = new MatchInstruction(SCHOOL_CLOSED_FIELDNAME, "false", null, false); + MatchInstruction matchSchoolId = new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, true); + MatchInstruction matchSchoolName = new MatchInstruction(SCHOOL_NAME_FIELDNAME, searchQuery, null, true); + MatchInstruction matchPostcode = new MatchInstruction(SCHOOL_POSTCODE_FIELDNAME, searchQuery, null, true); + + BooleanInstruction matchInstruction = new BooleanInstruction(); + matchInstruction.must(excludeClosedSchools); + matchInstruction.should(matchSchoolId); + matchInstruction.should(matchSchoolName); + matchInstruction.should(matchPostcode); + + List schoolSearchResults = searchProvider.nestedMatchSearch(SCHOOLS_INDEX_BASE, + SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), 0, queryLimit, matchInstruction, null, null).getResults(); List resultList = Lists.newArrayList(); for (String schoolString : schoolSearchResults) { From 7c7c2574bcd3f0dec7715e3a0b61e970cd664189 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 29 Apr 2026 14:16:58 +0100 Subject: [PATCH 05/19] Remove deprecated fuzzySearch method This is no longer used anywhere now that the school list reader uses a BooleanInstruction --- .../segue/search/ElasticSearchProvider.java | 70 ------------------- .../cl/dtg/segue/search/ISearchProvider.java | 32 --------- 2 files changed, 102 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/search/ElasticSearchProvider.java b/src/main/java/uk/ac/cam/cl/dtg/segue/search/ElasticSearchProvider.java index 41dfc70dda..a91f5cf4b3 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/search/ElasticSearchProvider.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/search/ElasticSearchProvider.java @@ -217,76 +217,6 @@ public ResultsWrapper nestedMatchSearch(final String indexBase, final St return this.executeBasicQuery(indexBase, indexType, query, startIndex, limit, sortOrder); } - @Override - @Deprecated - public ResultsWrapper fuzzySearch(final String indexBase, final String indexType, final String searchString, - final Integer startIndex, final Integer limit, - @Nullable final Map> fieldsThatMustMatch, - @Nullable final Map filterInstructions, - final String... fields) throws SegueSearchException { - if (null == indexBase || null == indexType || null == searchString || null == fields) { - log.warn("A required field is missing. Unable to execute search."); - return null; - } - - BoolQuery.Builder masterQuery; - if (null != fieldsThatMustMatch) { - masterQuery = new BoolQuery.Builder().must(this.generateBoolMatchQuery(this.convertToBoolMap(fieldsThatMustMatch))._toQuery()); - } else { - masterQuery = new BoolQuery.Builder(); - } - - BoolQuery.Builder query = new BoolQuery.Builder(); - Set boostFields = ImmutableSet.of("id", "title", "tags"); - - List searchTerms = Lists.newArrayList(); - searchTerms.addAll(Arrays.asList(searchString.split(" "))); - if (searchTerms.size() > 1) { - searchTerms.add(searchString); - } - - for (String f : fields) { - float boost = boostFields.contains(f) ? 2f : 1f; - - for (String searchTerm : searchTerms) { - Query initialFuzzySearch = MatchQuery.of(m -> m - .field(f) - .query(searchTerm) - .fuzziness("AUTO") - .prefixLength(0) - .boost(boost) - )._toQuery(); - query.should(initialFuzzySearch); - - Query regexSearch = WildcardQuery.of(r -> r - .field(f) - .value("*" + searchTerm + "*") - .boost(boost) - )._toQuery(); - query.should(regexSearch); - } - } - - // this query is just a bit smarter than the regex search above. - Query multiMatchPrefixQuery = MultiMatchQuery.of(mm -> mm - .query(searchString) - .fields(Arrays.asList(fields)) - .type(TextQueryType.PhrasePrefix) - .prefixLength(2) - .boost(2.0f) - )._toQuery(); - query.should(multiMatchPrefixQuery); - - masterQuery.must(query.build()._toQuery()); - - if (filterInstructions != null) { - masterQuery.filter(generateFilterQuery(filterInstructions)); - } - - Query finalQuery = masterQuery.build()._toQuery(); - return this.executeBasicQuery(indexBase, indexType, finalQuery, startIndex, limit); - } - @Override public ResultsWrapper termSearch(final String indexBase, final String indexType, final String searchTerm, final String field, final int startIndex, final int limit, diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/search/ISearchProvider.java b/src/main/java/uk/ac/cam/cl/dtg/segue/search/ISearchProvider.java index 68c13a9d28..14c587c051 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/search/ISearchProvider.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/search/ISearchProvider.java @@ -77,38 +77,6 @@ ResultsWrapper matchSearch( @Nullable final Map filterInstructions ) throws SegueSearchException; - /** - * Executes a fuzzy search on an array of fields and will consider the fieldsThatMustMatchMap. - * - * This method should prioritise exact prefix matches and then fill it with fuzzy ones. - * - * @param indexBase - * - the base string for the name of the index - * @param indexType - * - the name of the type of document being searched for - * @param searchString - * - the string to use for fuzzy matching - * @param startIndex - * - e.g. 0 for the first set of results - * @param limit - * - the maximum number of results to return -1 will attempt to return all results. - * @param fieldsThatMustMatch - * - Map of Must match field -> value - * @param filterInstructions - * - post search filter instructions e.g. remove content of a certain type. - * @param fields - * - array (var args) of fields to search using the searchString - * @return results - * @deprecated in favour of {@code BooleanInstruction}-based searches. - */ - @Deprecated - ResultsWrapper fuzzySearch( - final String indexBase, final String indexType, final String searchString, - final Integer startIndex, final Integer limit, final Map> fieldsThatMustMatch, - @Nullable final Map filterInstructions, - final String... fields - ) throws SegueSearchException; - ResultsWrapper nestedMatchSearch( final String indexBase, final String indexType, final Integer startIndex, final Integer limit, @NotNull final BooleanInstruction matchInstruction, @Nullable Long randomSeed, From 19f96fadbc4cd1a3e673944ab9010e0f766a5326 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Thu, 30 Apr 2026 15:31:08 +0100 Subject: [PATCH 06/19] Fix school ID Elasticsearch query The new schoolId fieldname contains a capital letter, so capitalisation matters where it didn't before. The raw fields suffix also breaks things for some reason, and since this is an exact match query anyway we don't need to explicitly disable things like stemming, so it should be safe to remove the suffix. --- .../uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index c08f7eb7f6..284b6627c8 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -147,8 +147,7 @@ public School findSchoolById(final String schoolId) throws UnableToIndexSchoolsE List matchingSchoolList; matchingSchoolList = searchProvider.findByExactMatch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), - SCHOOL_ID_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, - schoolId, 0, DEFAULT_RESULTS_LIMIT, null).getResults(); + SCHOOL_ID_FIELDNAME, schoolId, 0, DEFAULT_RESULTS_LIMIT, null).getResults(); if (matchingSchoolList.isEmpty()) { return null; From 4ca37f27554e642ddf609e8fd62721fb8f4d6ebc Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Thu, 30 Apr 2026 15:52:22 +0100 Subject: [PATCH 07/19] Exclude "excluded" schools when searching by school name/postcode --- .../segue/dao/schools/SchoolListReader.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 284b6627c8..5008e44f21 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -76,7 +76,7 @@ public SchoolListReader(final ISearchProvider searchProvider) { } /** - * findSchoolByNameOrPostCode. + * findSchoolByNameOrPostCode. Excludes schools marked as closed or excluded. * * @param searchQuery * - school to search for - either name or postcode. @@ -94,16 +94,14 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab Integer queryLimit = limit == null ? DEFAULT_RESULTS_LIMIT : limit; - MatchInstruction excludeClosedSchools = new MatchInstruction(SCHOOL_CLOSED_FIELDNAME, "false", null, false); - MatchInstruction matchSchoolId = new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, true); - MatchInstruction matchSchoolName = new MatchInstruction(SCHOOL_NAME_FIELDNAME, searchQuery, null, true); - MatchInstruction matchPostcode = new MatchInstruction(SCHOOL_POSTCODE_FIELDNAME, searchQuery, null, true); - BooleanInstruction matchInstruction = new BooleanInstruction(); - matchInstruction.must(excludeClosedSchools); - matchInstruction.should(matchSchoolId); - matchInstruction.should(matchSchoolName); - matchInstruction.should(matchPostcode); + // Exclude excluded/closed schools + matchInstruction.must(new MatchInstruction(SCHOOL_EXCLUDED_FIELDNAME, "false", null, false)); + matchInstruction.must(new MatchInstruction(SCHOOL_CLOSED_FIELDNAME, "false", null, false)); + // Attempt to match on school ID, name & postcode + matchInstruction.should(new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, true)); + matchInstruction.should(new MatchInstruction(SCHOOL_NAME_FIELDNAME, searchQuery, null, true)); + matchInstruction.should(new MatchInstruction(SCHOOL_POSTCODE_FIELDNAME, searchQuery, null, true)); List schoolSearchResults = searchProvider.nestedMatchSearch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), 0, queryLimit, matchInstruction, null, null).getResults(); From f9dc43fa7cab19fea4a98b45cf5c1942592ff605 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Fri, 1 May 2026 14:05:58 +0100 Subject: [PATCH 08/19] Allow school searches to be filtered by country code --- .../cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java | 5 +++-- .../ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java | 8 +++++++- .../uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java index 55061c82e7..99b14a700f 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java @@ -84,7 +84,8 @@ public SchoolLookupServiceFacade(final SchoolListReader schoolListReader) { @GZIP @Operation(summary = "List all schools matching provided criteria.") public Response schoolSearch(@Context final Request request, @QueryParam("query") final String searchQuery, - @QueryParam("schoolId") final String schoolId, @QueryParam("limit") final Integer limit) { + @QueryParam("countryCode") final String countryCode, @QueryParam("schoolId") final String schoolId, + @QueryParam("limit") final Integer limit) { if ((null == searchQuery || searchQuery.isEmpty()) && (null == schoolId || schoolId.isEmpty())) { return new SegueErrorResponse(Status.BAD_REQUEST, "You must provide a search query or school ID") @@ -112,7 +113,7 @@ public Response schoolSearch(@Context final Request request, @QueryParam("query" if (schoolId != null && !schoolId.isEmpty()) { list = Collections.singletonList(schoolListReader.findSchoolById(schoolId)); } else { - list = schoolListReader.findSchoolByNameOrPostCode(searchQuery, limit); + list = schoolListReader.findSchoolByNameOrPostCode(searchQuery, countryCode, limit); } } catch (UnableToIndexSchoolsException | SegueSearchException | IOException e) { diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 5008e44f21..1dc039141a 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -86,7 +86,8 @@ public SchoolListReader(final ISearchProvider searchProvider) { * @throws UnableToIndexSchoolsException * - if there is an error access the index of schools. */ - public List findSchoolByNameOrPostCode(final String searchQuery, @Nullable final Integer limit) throws UnableToIndexSchoolsException, SegueSearchException { + public List findSchoolByNameOrPostCode(final String searchQuery, @Nullable final String countryCode, + @Nullable final Integer limit) throws UnableToIndexSchoolsException, SegueSearchException { if (!this.ensureSchoolList()) { log.error("Unable to ensure school search cache."); throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated"); @@ -98,6 +99,11 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab // Exclude excluded/closed schools matchInstruction.must(new MatchInstruction(SCHOOL_EXCLUDED_FIELDNAME, "false", null, false)); matchInstruction.must(new MatchInstruction(SCHOOL_CLOSED_FIELDNAME, "false", null, false)); + // Compulsorily match country code, if provided. Needs to be a raw field since GB- country codes contain hyphens. + if (null != countryCode && !countryCode.isEmpty()) { + matchInstruction.must(new MatchInstruction(SCHOOL_COUNTRY_CODE_FIELDNAME + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, + countryCode, null, false)); + } // Attempt to match on school ID, name & postcode matchInstruction.should(new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, true)); matchInstruction.should(new MatchInstruction(SCHOOL_NAME_FIELDNAME, searchQuery, null, true)); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java index c0c39f4168..dd418d0c0b 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java @@ -61,7 +61,7 @@ public class ElasticSearchIndexer extends ElasticSearchProvider { public ElasticSearchIndexer(final ElasticsearchClient searchClient) { super(searchClient); rawFieldsListByType.put("content", Lists.newArrayList("id", "title", "subtitle")); - rawFieldsListByType.put("school", Lists.newArrayList("urn")); + rawFieldsListByType.put("school", Lists.newArrayList("schoolId", "countryCode")); nestedFieldsByType.put("content", Lists.newArrayList("audience")); } From 75d9e6a9a0d8f056230fc52d7454c151b992ef00 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Fri, 1 May 2026 14:12:39 +0100 Subject: [PATCH 09/19] Remove reference to URN --- src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java index 4fd1a0bf9e..fce86d8d03 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java @@ -265,6 +265,6 @@ public boolean equals(final Object obj) { @Override public String toString() { - return "School [urn=" + schoolId + ", name=" + schoolName + ", postcode=" + postalCode + "]"; + return "School [id=" + schoolId + ", name=" + schoolName + ", postcode=" + postalCode + "]"; } } From 620122f269246a3be2b8d638f2e316c474ba710e Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Fri, 1 May 2026 14:15:17 +0100 Subject: [PATCH 10/19] Update Javadoc --- .../ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 1dc039141a..52ae0fa880 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -76,10 +76,12 @@ public SchoolListReader(final ISearchProvider searchProvider) { } /** - * findSchoolByNameOrPostCode. Excludes schools marked as closed or excluded. + * Search for schools by ID, name or postcode. Filterable by country. Excludes schools marked as closed or excluded. * * @param searchQuery - * - school to search for - either name or postcode. + * - school to search for - either ID, name or postcode. + * @param countryCode + * - country code to filter by. * @param limit * - the number of results to return. * @return list of schools matching the criteria or an empty list. From 1a6e0b75758e42f27eb5443039f52d5e4ec22552 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Fri, 1 May 2026 14:20:12 +0100 Subject: [PATCH 11/19] Remove unused schoolId raw ES field --- .../java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java index dd418d0c0b..7c88a1d4a8 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java @@ -61,7 +61,7 @@ public class ElasticSearchIndexer extends ElasticSearchProvider { public ElasticSearchIndexer(final ElasticsearchClient searchClient) { super(searchClient); rawFieldsListByType.put("content", Lists.newArrayList("id", "title", "subtitle")); - rawFieldsListByType.put("school", Lists.newArrayList("schoolId", "countryCode")); + rawFieldsListByType.put("school", Lists.newArrayList("countryCode")); nestedFieldsByType.put("content", Lists.newArrayList("audience")); } From ab350c6948f08e6a4ea2b8a36e391745baf328ea Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 12 Aug 2026 09:51:21 +0100 Subject: [PATCH 12/19] Fix school search by ID Since we are now using a MatchInstruction here, and school ID is no longer indexed as a raw field, there should no longer be any processing done on the school ID fieldname here. --- .../uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index fed01f6ee2..bb99135b83 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -150,7 +150,7 @@ public School findSchoolById(final String schoolId) throws UnableToIndexSchoolsE throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated"); } - MatchInstruction searchInstruction = new MatchInstruction(SCHOOL_ID_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, schoolId); + MatchInstruction searchInstruction = new MatchInstruction(SCHOOL_ID_FIELDNAME, schoolId); List matchingSchoolList = searchProvider.nestedMatchSearch(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), 0, DEFAULT_RESULTS_LIMIT, searchInstruction, null, null From 35173fbfa14feaa4866518db1de6ca9cc1bf8238 Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 12 Aug 2026 10:41:50 +0100 Subject: [PATCH 13/19] Update school data sources --- src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java index fce86d8d03..20f1d45c5d 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/users/School.java @@ -32,7 +32,8 @@ public class School { * Enum to represent where this school object was created. */ public enum SchoolDataSource { - GOVERNMENT_UK, GOVERNMENT_IE, GOVERNMENT_SCT, GOVERNMENT_SCT_IND, GOVERNMENT_WLS, GOVERNMENT_NIR, USER_ENTERED; + GOVERNMENT_UK, GOVERNMENT_SCT, GOVERNMENT_SCT_COL, GOVERNMENT_SCT_IND, GOVERNMENT_WLS, GOVERNMENT_WLS_FE, + GOVERNMENT_NIR, GOVERNMENT_IE, USER_ENTERED; @Override public String toString() { From 4e393289ea0f20871698c3dcd4f59ef9e7a183ad Mon Sep 17 00:00:00 2001 From: Alex Lewin Date: Wed, 12 Aug 2026 10:51:48 +0100 Subject: [PATCH 14/19] Remove school missing postcode warning This prints lots of warnings when running ETL, and isn't useful. --- src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java index 4be7060959..7e4503d46f 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java @@ -138,10 +138,6 @@ private synchronized List loadAndBuildSchoolList() throws UnableToIndexS "true".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]), source); - if (null == schoolToSave.getPostalCode() || schoolToSave.getPostalCode().isEmpty()) { - log.warn("School with missing postcode! School ID: {}", schoolToSave.getSchoolId()); - } - schools.add(schoolToSave); } catch (IndexOutOfBoundsException e) { // This happens when the school does not have the required data From cca13bf30a6f49395419eaf68847253faebfc964 Mon Sep 17 00:00:00 2001 From: James Sharkey Date: Wed, 12 Aug 2026 14:50:40 +0100 Subject: [PATCH 15/19] Improve input validation on school search endpoint --- .../uk/ac/cam/cl/dtg/segue/api/Constants.java | 1 + .../segue/api/SchoolLookupServiceFacade.java | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java index 5c90c50661..8e634ef238 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java @@ -443,6 +443,7 @@ public enum SegueServerLogType implements LogType { public static final String DEFAULT_SEARCH_RESULT_LIMIT_AS_STRING = "25"; public static final Integer MAX_SEARCH_RESULT_LIMIT = 350; + public static final Integer MAX_SCHOOLS_RESULT_LIMIT = 50; public static final Integer SEARCH_TEXT_CHAR_LIMIT = 1000; public static final Integer CONTACT_FORM_CHAR_LIMIT = 5000; diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java index 99b14a700f..518ac0c251 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/SchoolLookupServiceFacade.java @@ -43,6 +43,8 @@ import java.util.Collections; import java.util.List; +import static uk.ac.cam.cl.dtg.segue.api.Constants.*; + /** * Segue School Lookup service. * @@ -87,9 +89,18 @@ public Response schoolSearch(@Context final Request request, @QueryParam("query" @QueryParam("countryCode") final String countryCode, @QueryParam("schoolId") final String schoolId, @QueryParam("limit") final Integer limit) { - if ((null == searchQuery || searchQuery.isEmpty()) && (null == schoolId || schoolId.isEmpty())) { - return new SegueErrorResponse(Status.BAD_REQUEST, "You must provide a search query or school ID") - .toResponse(); + if (null != limit && limit > MAX_SCHOOLS_RESULT_LIMIT) { + return SegueErrorResponse.getBadRequestResponse("Too many schools requested!"); + } + + boolean queryEmpty = searchQuery == null || searchQuery.isEmpty(); + boolean schoolIdEmpty = schoolId == null || schoolId.isEmpty(); + boolean countryCodeEmpty = countryCode == null || countryCode.isEmpty(); + if ((queryEmpty && schoolIdEmpty) || (!queryEmpty && !schoolIdEmpty)) { + return SegueErrorResponse.getBadRequestResponse("You must provide either a search query, or a school ID."); + } + if (!schoolIdEmpty && !countryCodeEmpty) { + return SegueErrorResponse.getBadRequestResponse("You cannot filter by country when searching by school ID."); } EntityTag etag = new EntityTag(schoolListReader.getDataLastModifiedDate()); From 8b6501714b2804444303c579ea3d93afda5605da Mon Sep 17 00:00:00 2001 From: James Sharkey Date: Wed, 12 Aug 2026 15:17:11 +0100 Subject: [PATCH 16/19] Do not fuzzy match school IDs when query searching We obviously need to match exactly when searching by ID, but I think we should also only match exactly when querying here too, to avoid loose matches affecting the score of results. --- .../uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index bb99135b83..5cef98a961 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -107,7 +107,7 @@ public List findSchoolByNameOrPostCode(final String searchQuery, @Nullab countryCode, null, false)); } // Attempt to match on school ID, name & postcode - matchInstruction.should(new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, true)); + matchInstruction.should(new MatchInstruction(SCHOOL_ID_FIELDNAME, searchQuery, null, false)); matchInstruction.should(new MatchInstruction(SCHOOL_NAME_FIELDNAME, searchQuery, null, true)); matchInstruction.should(new MatchInstruction(SCHOOL_POSTCODE_FIELDNAME, searchQuery, null, true)); From 47efc4eed51ec4e77c22f16e0e0f79c456aed2cd Mon Sep 17 00:00:00 2001 From: James Sharkey Date: Thu, 13 Aug 2026 13:50:32 +0100 Subject: [PATCH 17/19] Consistently use enums to reference content indices --- .../cam/cl/dtg/segue/etl/ElasticSearchIndexer.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java index 7c88a1d4a8..8281b3f9c1 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/ElasticSearchIndexer.java @@ -27,7 +27,6 @@ import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import uk.ac.cam.cl.dtg.segue.api.Constants; import uk.ac.cam.cl.dtg.segue.search.ElasticSearchProvider; import uk.ac.cam.cl.dtg.segue.search.SegueSearchException; @@ -40,6 +39,8 @@ import java.util.Map; import java.util.function.Function; +import static uk.ac.cam.cl.dtg.segue.api.Constants.*; + /** * Created by Ian on 17/10/2016. */ @@ -60,9 +61,9 @@ public class ElasticSearchIndexer extends ElasticSearchProvider { @Inject public ElasticSearchIndexer(final ElasticsearchClient searchClient) { super(searchClient); - rawFieldsListByType.put("content", Lists.newArrayList("id", "title", "subtitle")); - rawFieldsListByType.put("school", Lists.newArrayList("countryCode")); - nestedFieldsByType.put("content", Lists.newArrayList("audience")); + rawFieldsListByType.put(CONTENT_INDEX_TYPE.CONTENT.toString(), Lists.newArrayList("id", "title", "subtitle")); + rawFieldsListByType.put(SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), Lists.newArrayList("countryCode")); + nestedFieldsByType.put(CONTENT_INDEX_TYPE.CONTENT.toString(), Lists.newArrayList("audience")); } @@ -319,11 +320,11 @@ private void sendMappingCorrections(final String typedIndex, final String indexT // Add mapping to specify raw, un-analyzed fields for (String fieldName : this.rawFieldsListByType.getOrDefault(indexType, Collections.emptyList())) { - log.debug("Sending raw mapping correction for {}." + Constants.UNPROCESSED_SEARCH_FIELD_SUFFIX, fieldName); + log.debug("Sending raw mapping correction for {}." + UNPROCESSED_SEARCH_FIELD_SUFFIX, fieldName); properties.put(fieldName, Property.of(p -> p .text(t -> t .fields( - Constants.UNPROCESSED_SEARCH_FIELD_SUFFIX, + UNPROCESSED_SEARCH_FIELD_SUFFIX, Property.of(k -> k.keyword(kf -> kf)) ) ) From fdbb0d60094be122e5c7ce8a03bebdfc2a14028d Mon Sep 17 00:00:00 2001 From: James Sharkey Date: Thu, 13 Aug 2026 13:55:28 +0100 Subject: [PATCH 18/19] Fix SchoolIndexer Checkstyle warnings --- .../cam/cl/dtg/segue/etl/SchoolIndexer.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java index 7e4503d46f..d9f3b80546 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java @@ -34,11 +34,11 @@ */ class SchoolIndexer { private static final Logger log = LoggerFactory.getLogger(SchoolIndexer.class); - private ElasticSearchIndexer es; - private ContentSubclassMapper mapper; - private String schoolsListPath; + private final ElasticSearchIndexer es; + private final ContentSubclassMapper mapper; + private final String schoolsListPath; - SchoolIndexer(ElasticSearchIndexer es, ContentSubclassMapper mapper, String schoolsListPath) { + SchoolIndexer(final ElasticSearchIndexer es, final ContentSubclassMapper mapper, final String schoolsListPath) { this.es = es; this.mapper = mapper; this.schoolsListPath = schoolsListPath; @@ -66,7 +66,7 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx for (School school : schoolList) { try { indexList.add(immutableEntry(school.getSchoolId(), objectMapper.writeValueAsString(school))); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { log.error("Unable to serialize the school object into json.", e); } } @@ -75,16 +75,16 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx try { es.indexObject(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.METADATA.toString(), objectMapper.writeValueAsString( ImmutableMap.of("lastModified", f.lastModified())), "sourceFile"); - } catch (SegueSearchException e) { + } catch (final SegueSearchException e) { log.error("Unable to index school list metadata.", e); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { log.error("Unable to serialise school list last modified date to JSON.", e); } try { es.bulkIndexWithIDs(SCHOOLS_INDEX_BASE, SCHOOLS_INDEX_TYPE.SCHOOL_SEARCH.toString(), indexList); log.info("School list index request complete."); - } catch (SegueSearchException e) { + } catch (final SegueSearchException e) { log.error("Unable to complete bulk index operation for schools list.", e); } @@ -139,17 +139,17 @@ private synchronized List loadAndBuildSchoolList() throws UnableToIndexS source); schools.add(schoolToSave); - } catch (IndexOutOfBoundsException e) { + } catch (final IndexOutOfBoundsException e) { // This happens when the school does not have the required data log.warn("Unable to load school into list due to missing required fields: {}", Arrays.toString(schoolArray)); } } - } catch (FileNotFoundException e) { + } catch (final FileNotFoundException e) { log.error("Unable to locate the file requested", e); throw new UnableToIndexSchoolsException("Unable to locate the file requested", e); - } catch (IOException e) { + } catch (final IOException e) { throw new UnableToIndexSchoolsException("Unable to load the file requested", e); - } catch (CsvValidationException e) { + } catch (final CsvValidationException e) { throw new UnableToIndexSchoolsException("Unable to parse the file requested", e); } From a43bf26ccd36c59bdeded8f542a3be13b4e2ef12 Mon Sep 17 00:00:00 2001 From: James Sharkey Date: Thu, 13 Aug 2026 13:59:56 +0100 Subject: [PATCH 19/19] Use new ES index name for new school list Since we have the old and new APIs running simultaneously, we can't afford to expunge the old school list index without bringing the old API offline - otherwise it will fail when it tries to read the school data. So create a new index name (and unique alias names!) for the new format data, so both can coexist. --- src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java | 2 +- src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java index 8e634ef238..65bf91d3c9 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java @@ -247,7 +247,7 @@ public enum BooleanOperator { AND, OR, NOT } - public static final String SCHOOLS_INDEX_BASE = "schools"; + public static final String SCHOOLS_INDEX_BASE = "schools_2026"; public enum SCHOOLS_INDEX_TYPE { METADATA("metadata"), SCHOOL_SEARCH("school"); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java index d9f3b80546..8fc0300fdd 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java @@ -91,7 +91,7 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx // Create an alias (could be anything) to prevent this schools index from being garbage-collected by ElasticSearchIndexer.expungeOldIndices List allSchoolTypes = Arrays.stream(SCHOOLS_INDEX_TYPE.values()) .map(SCHOOLS_INDEX_TYPE::toString).collect(Collectors.toList()); - es.addOrMoveIndexAlias("schools-latest", SCHOOLS_INDEX_BASE, allSchoolTypes); + es.addOrMoveIndexAlias(String.format("%s_latest", SCHOOLS_INDEX_BASE), SCHOOLS_INDEX_BASE, allSchoolTypes); } /**