diff --git a/code/generators/html/refiner.py b/code/generators/html/refiner.py index 8c3a8c2f5..b166ce85b 100644 --- a/code/generators/html/refiner.py +++ b/code/generators/html/refiner.py @@ -228,6 +228,7 @@ class HtmlRefiner: "style", "title", "textarea", + "h1", # exclude so there are no self links } _HEADING_TAGS = ("h1", "h2", "h3", "h4", "h5", "h6") diff --git a/code/name_corrections.json b/code/name_corrections.json new file mode 100644 index 000000000..b8005f31b --- /dev/null +++ b/code/name_corrections.json @@ -0,0 +1,213 @@ +{ + "input_corrections": { + "1PN": "1PN", + "3PN": "3PN", + "3rd": "3rd", + "60U": "60U", + "AboveGround": "Aboveground", + "ANNODIZED": "Anodized", + "ASPHIXIATION": "Asphyxiation", + "ASSIGNEE": "Assignee", + "ASSIGNOR": "Assignor", + "ATOMIZING": "Atomizing", + "BALISE": "Balise", + "BELOWGROUND": "Belowground", + "BEZIER": "Bezier", + "BIFUEL": "Bi-Fuel", + "BILINEAR": "Bilinear", + "BLOSSCURVE": "Bloss Curve", + "BOLLARD": "Bollard", + "BRAZED": "Brazed", + "Breakinging": "Breaking", + "BREECHING": "Breeching", + "CATENARY": "Catenary", + "CHAMFER": "Chamfer", + "CLASS0I": "Class 0I", + "CLOTHOID": "Clothoid", + "COMMERICAL": "Commercial", + "COMPENSATOR": "Compensator", + "Complementaryfunction": "Complementary Function", + "CONVECTOR": "Convector", + "CR3": "CR3", + "CRANEWAY": "Craneway", + "DD0": "Dd0", + "DD6": "Dd6", + "DEIONIZING": "Deionizing", + "DERAILER": "Derailer", + "DEVIATOR": "Deviator", + "DIABOLO": "Diabolo", + "DIAZED": "Diazed", + "DIFFUSERS": "Diffusers", + "DISCONNECTOR": "Disconnector", + "DISHWASHER": "Dishwasher", + "DISPLACMENT": "Displacement", + "DIVERTER": "Diverter", + "DY11": "Dy11", + "DY5": "Dy5", + "DZ0": "Dz0", + "DZ6": "Dz6", + "ELASTOMERIC": "Elastomeric", + "ELEMENTED": "Elemented", + "EVALUATOR": "Evaluator", + "EXPENSED": "Expensed", + "FairLead": "Fairlead", + "FENESTRATION": "Fenestration", + "Flowrate": "Flow Rate", + "GALVANISED": "Galvanised", + "GAMBREL": "Gambrel", + "GUDING": "Guding", + "HELMERTCURVE": "Helmert Curve", + "HERSE": "Herse", + "HUMIDISTAT": "Humidistat", + "HYDRONIC": "Hydronic", + "i2t": "I2t", + "INSERTABLE": "Insertable", + "INSITU": "In Situ", + "INTERPOLATIVE": "Interpolative", + "ISOCONTOUR": "Isocontour", + "ISOLUX": "Isolux", + "LASERSCAN": "Laserscan", + "LEFTHAND": "Left Hand", + "Lfunction": "L-Function", + "LIMITER": "Limiter", + "MINIZED": "Minized", + "MULTIPASS": "Multi-Pass", + "N20": "N2O", + "NEOZED": "Neozed", + "Numberof": "Number of", + "OM1": "OM1", + "OM2": "OM2", + "OM3": "OM3", + "OM4": "OM4", + "OM5": "OM5", + "OVERTURING": "Overturning", + "PEDOLOGICAL": "Pedological", + "PICTORAL": "Pictoral", + "POLYPHASE": "Polyphase", + "POSITIONER": "Positioner", + "PRECAST": "Precast", + "PRESTRESSING": "Prestressing", + "PREVENTER": "Preventer", + "PROPELLOR": "Propellor", + "PURLIN": "Purlin", + "R320CR": "R320CR", + "RECHARGER": "Recharger", + "REDUCER": "Reducer", + "REVETMENT": "Revetment", + "SFunction": "S-Function", + "SIGANL": "Signal", + "SINGLESCREW": "Single Screw", + "SLEEVING": "Sleeving", + "SPANDREL": "Spandrel", + "STRUCKFALLINFOBJECT": "Struck Falling Object", + "SUBMASTER": "Submaster", + "SUBRACK": "Subrack", + "SYMETRIC": "Symmetric", + "TELECOMMAND": "Telecommand", + "TENSIONER": "Tensioner", + "TENSIONING": "Tensioning", + "THEAD": "T-Head", + "THEODOLITE": "Theodolite", + "TRANSVERSAL": "Transversal", + "TROCHOIDAL": "Trochoidal", + "TWINSCREW": "Twin Screw", + "UAcurves": "UA Curves", + "UNCRIMPED": "Uncrimped", + "UNPRICED": "Unpriced", + "UPGRADEABLETO": "Upgradeable To", + "VARISTOR": "Varistor", + "VEHICLESTOP": "Vehicle Stop", + "WASHER": "Washer", + "WINDER": "Winder", + "Worksurface": "Work Surface", + "YY0": "Yy0", + "YY6": "Yy6", + "YZ11": "Yz11", + "YZ5": "Yz5" + }, + + "token_render": { + "2d": "2D", + "3d": "3D", + "ac": "AC", + "am": "AM", + "and": "and", + "at": "at", + "bm": "BM", + "cdma": "CDMA", + "cgt": "CGT", + "chp": "CHP", + "co": "CO", + "co2": "CO2", + "crs": "CRS", + "cwr": "CWR", + "dc": "DC", + "dx": "DX", + "ep": "EP", + "ffl": "FFL", + "for": "for", + "gfa": "GFA", + "gprs": "GPRS", + "gps": "GPS", + "gsm": "GSM", + "hcfc": "HCFC", + "hvac": "HVAC", + "i2t": "I2t", + "ic": "IC", + "icm": "ICM", + "icn": "ICN", + "ics": "ICS", + "icu": "ICU", + "icw": "ICW", + "id": "ID", + "ii": "II", + "iii": "III", + "iot": "IoT", + "ip": "IP", + "ipi": "IPI", + "ir": "IR", + "led": "LED", + "lrm": "LRM", + "lte": "LTE", + "mp": "MP", + "msc": "MSC", + "n2o": "N2O", + "ocl": "OCL", + "ocs": "OCS", + "of": "of", + "ol": "OL", + "ole": "OLE", + "oled": "OLED", + "or": "or", + "ph": "pH", + "ppm": "ppm", + "ra": "RA", + "roi": "ROI", + "rru": "RRU", + "rs": "RS", + "ru": "RU", + "sc": "SC", + "scdma": "SCDMA", + "ssl": "SSL", + "std": "STD", + "td": "TD", + "teu": "TEU", + "tmp": "TMP", + "tn": "TN", + "to": "to", + "tv": "TV", + "tx": "TX", + "ty": "TY", + "tz": "TZ", + "ua": "UA", + "uri": "URI", + "url": "URL", + "ups": "UPS", + "usb": "USB", + "utran": "UTRAN", + "wcdma": "WCDMA", + "with": "with", + "www": "WWW", + "yd": "YD" + } +} diff --git a/code/name_improve.py b/code/name_improve.py index 10c39f659..a1a09b127 100644 --- a/code/name_improve.py +++ b/code/name_improve.py @@ -1,149 +1,32 @@ -from nltk import PorterStemmer -from reversestem import unstem -import re -import os import json -from copy import deepcopy -import logging -from extract_definition import MARKER +import re +from pathlib import Path +import wordninja -type_words_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), ".", "type_words.json") -) -with open(type_words_path, "r", encoding="utf-8") as file: - data = json.load(file) -type_words = sorted(data, key=len)[::-1] - -logging.basicConfig(level=logging.INFO) - -ps = PorterStemmer() - -ALL_CAPS = [ - "ups", - "gprs", - "rs", - "am", - "gps", - "dc", - "tn", - "url", - "ac", - "co", - "co2", - "chp", - "id", - "led", - "oled", - "ole", - "gfa", - "tv", - "msc", - "ppm", - "iot", - "ocl", - "lrm", - "cgt", - "teu", - "tmp", - "std", - "gsm", - "cdma", - "lte", - "td", - "scdma", - "wcdma", - "sc", - "mp", - "bm", - "ol", - "ep", - "ir", - "www", - "ip", - "ph", - "usb", - "ii", - "iii", - "url", - "uri", - "ssl", - "ffl", - "ty", - "tz", - "tx", - "ipi", - "ics", -] - -SMALL_CAPS = ["for", "of", "and", "to", "with", "or", "at"] - -SUBSTITUTIONS = { - "Rel ": "Relation: ", - "Min ": "Minimum ", - "Max ": "Maximum ", - " Temp": " Temperature", - "Qto_": "Quantity set: ", - "Pset_": "Property set: ", -} - - -def normalise(s): - """format the text by spliting camelCase into separate words and replacing special prefixes.""" - s = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", s) - for k, v in SUBSTITUTIONS.items(): - if k in s: - s = re.sub(k, v, s) - return re.sub(MULTIPLE_SPACE_PATTERN, " ", s).strip() - - -def caps_control(s): - """Turn special words to all caps or small caps using hard-coded lists. - E.g. Usb -> USB, With -> with""" - s1 = s.split() - if isinstance(s1, list): - for part in s1: - for a in ALL_CAPS: - if part.lower() == a: - s = re.sub(part, a.upper(), s) - for b in SMALL_CAPS: - if part.lower() == b: - s = re.sub(part, b.lower(), s) - return s +from extract_definition import MARKER +_corrections = json.loads( + Path(__file__).with_name("name_corrections.json").read_text("utf-8") +) +INPUT_CORRECTIONS = _corrections["input_corrections"] +TOKEN_RENDER = {k.lower(): v for k, v in _corrections["token_render"].items()} +CORRECTION_PATTERN = re.compile( + "|".join(re.escape(k) for k in sorted(INPUT_CORRECTIONS, key=len, reverse=True)) +) -def split_at_word(w, s): - """Try spliting the string s with the word w. Return string s splited with whitespaces.""" - if w != "" and ((ps.stem(w) in s.lower()) or (w in s.lower())): - # to_keep=False - # for wtk in words_to_keep: - # if wtk in s.lower(): - # to_keep=True - # if not to_keep: - if w in s.lower(): - s = re.sub(w, " " + w + " ", s.lower()) - else: - vs = unstem(ps.stem(w)) - if isinstance(vs, dict): - # turn dict to flat list - vsl = [] - for key, val in vs.items(): - vsl.append(key) - if val: - for v in val: - vsl.append(v) - vs = vsl - if vs: - vs.append(ps.stem(w)) - vs = sorted(vs, key=len)[::-1] - for v in vs: - if v in s.lower(): - s = re.sub(w, " " + w + " ", s.lower()) - break - if isinstance(s, list): - s = " ".join(s) - return s +AGGREGATION_BOUND = re.compile(r"_\w+\[") # IfcComplexNumber_A[1:2] -> IfcComplexNumber +CAMEL_BOUNDARY = re.compile( + r"(?<=[a-z])(?=[A-Z0-9])" + r"|(?<=[A-Z][A-Z])(?=[0-9])" + r"|(?<=[0-9])(?=[a-z])" + r"|(?<=[0-9])(?=[A-Z][A-Za-z])" + r"|(?<=[A-Z])(?=[A-Z][a-z])" +) +MAX_ACRONYM_LEN = 4 +PLURAL_TAIL = re.compile(r"^[A-Z](s|es)$") +ACRONYM_PLURAL = re.compile(rf"^[A-Z]{{2,{MAX_ACRONYM_LEN}}}(s|es)$") MULTIPLE_LINEBREAK_PATTERN = re.compile("\n+") MULTIPLE_SPACE_PATTERN = re.compile(r"\s+") @@ -151,16 +34,11 @@ def split_at_word(w, s): CURLY_BRACKET_PATTERN = re.compile( "{.*?}.*" ) # also removes all text after curly brackets -FIGURE_PATTERN = re.compile( - "[^.,;]*(Figure|the figure)[^.,;]*" -) # removes the sentence when it mentiones a Figure. -LIST_PATTERN = re.compile( - r"[^.,;]*:\s?" -) # removes the last sentence when it ends with a colon. -# CHANGE_LOG_PATTERN = re.compile(r'\{\s*\.change\-\w+\s*\}.+', flags=re.DOTALL) - +FIGURE_PATTERN = re.compile("[^.,;]*(Figure|the figure)[^.,;]*") +LIST_PATTERN = re.compile(r"[^.,;]*:\s?") -replacements = [ +REPLACEMENTS = [ + (MULTIPLE_LINEBREAK_PATTERN, "; "), (re.compile(r"\*{2}"), " "), (re.compile(r":(?!\s)"), ": "), (re.compile(r"SELF\\"), ""), @@ -168,7 +46,6 @@ def split_at_word(w, s): re.compile(r"(? ['IDs']; ['CO','2'] -> ['co2'] + out, i = [], 0 + while i < len(tokens): + cur, nxt = tokens[i], tokens[i + 1] if i + 1 < len(tokens) else None + if ( + nxt + and cur.isalpha() + and cur.isupper() + and len(cur) <= MAX_ACRONYM_LEN + and PLURAL_TAIL.match(nxt) + ): + out.append(cur + nxt) + i += 2 + elif nxt and (cur + nxt).lower() in TOKEN_RENDER: + out.append((cur + nxt).lower()) + i += 2 + else: + out.append(cur) + i += 1 + return out + + +def _render(tokens): + rendered = [] + for t in tokens: + override = TOKEN_RENDER.get(t.lower()) + if override is not None: + if override.islower() and not rendered: + rendered.append(override.capitalize()) + else: + rendered.append(override) + elif t.isdigit() or ACRONYM_PLURAL.match(t): + rendered.append(t) + else: + rendered.append(t.capitalize()) + return " ".join(rendered) + + +def _merge_plural_s(pieces): + # ['EXCHANGER', 'S'] -> ['EXCHANGERS'] + merged = [] + for piece in pieces: + if piece.lower() == "s" and merged: + merged[-1] += piece + else: + merged.append(piece) + return merged + + +def _render_span(s): + tokens = [] + for tok in _split_identifier(s): + if ( + tok.isalpha() + and tok.isupper() + and len(tok) > MAX_ACRONYM_LEN + and tok.lower() not in TOKEN_RENDER + ): + tokens.extend(_merge_plural_s(wordninja.split(tok))) + else: + tokens.append(tok) + return _render(_rejoin_acronym_fragments(tokens)) + + +def _split_into_literal_and_process_parts(s): + parts, last = [], 0 + for m in CORRECTION_PATTERN.finditer(s): + if m.start() > last: + parts.append(("process", s[last : m.start()])) + parts.append(("literal", INPUT_CORRECTIONS[m.group(0)])) + last = m.end() + if last < len(s): + parts.append(("process", s[last:])) + return parts + + +def name_improve(s): + s = AGGREGATION_BOUND.split(s)[0] + out = [] + for kind, val in _split_into_literal_and_process_parts(s): + if kind == "literal": + out.append(val) + else: + rendered = _render_span(val) + if rendered: + out.append(rendered) + return " ".join(out).strip() + + def remove_unwanted(s): - for pat, subs in replacements: + for pat, subs in REPLACEMENTS: s = re.sub(pat, subs, s) return s def clean(s): - """format the text by removing unwanted characters.""" - # remove all redundant characters (except those listed): - s = re.sub(MULTIPLE_LINEBREAK_PATTERN, "; ", s) s = remove_unwanted(s) cleaned = "".join( - ["", c][c.isalnum() or c in ":.,()-+ —=;α°_/!?$%@<>'\"\\*"] for c in s + c for c in s if c.isalnum() or c in ":.,()-+ —=;α°_/!?$%@<>'\"\\*" ) - p = re.compile('"') - cleaned = re.sub(p, "'", cleaned).strip() + cleaned = cleaned.replace('"', "'") return re.sub(MULTIPLE_SPACE_PATTERN, " ", cleaned).strip() -def split_words(s): - """Split the phrase using the hard-coded list of words found in enumerations - to split the ALLCAPS phrases into individual words.""" - found_words = [] - for w in type_words: - # skip if word is contained in previous words (e.g. EXCHANGE in EXCHANGER) - previously_found = False - for fw in found_words: - if w in fw: - previously_found = True - if not previously_found: - s2 = deepcopy(s) - s2 = split_at_word(w, s2) - if s2 != s: - found_words.append(w) - s = s2 - s = re.sub("_", " ", s) - s = re.sub(r"\s+", " ", s) - return s.strip() - - -def name_improve(s): - s = re.split(r"_\w\[", s)[0] - if s.lower().startswith("ifc"): - s = s[3:] - return caps_control(split_words(clean(normalise(s))).title()) - - def definition_improve(s): return clean(s.split(MARKER.strip(), 1)[0].strip()) diff --git a/code/requirements.txt b/code/requirements.txt index dbe04a63f..84d88ca46 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -29,6 +29,7 @@ networkx # Data / utils numpy tabulate +wordninja==2.0.0 deepdiff hilbertcurve==1.0.5 python-dotenv diff --git a/code/templates/search.html b/code/templates/search.html index f5a0e668b..643ad7598 100644 --- a/code/templates/search.html +++ b/code/templates/search.html @@ -1,4 +1,8 @@ {% extends "main.html" %} + +{# The search page has its own (larger) search form; suppress/overwrite the sidebar one from main.html. #} +{% block searchbar %}{% endblock %} + {% block pagecontent %}