diff --git a/.env b/.env
index 49173870..58624d86 100644
--- a/.env
+++ b/.env
@@ -72,12 +72,6 @@ CHECK_ALL_BOOKS_FOR_LANGUAGE=true
# they all are available by cloning, hence the value false.
DOWNLOAD_ASSETS=false
-# Setting to false will cause interleave by verse layout to put each
-# TW word associated with the verse in a horizontal comma delimited
-# list, otherwise it will put each TW word in a list with one word per
-# line.
-TW_WORD_LIST_VERTICAL=false
-
# File lock duration when cloning and potentially removing git repos
LOCK_TIMEOUT_SECONDS=300
diff --git a/Makefile b/Makefile
index bc2fc143..2692d81d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,21 @@
+.PHONY: startdocker
+startdocker:
+ @if docker info >/dev/null 2>&1; then \
+ echo "Docker is already running."; \
+ else \
+ echo "Starting Docker..."; \
+ if [ "$$(uname)" = "Darwin" ]; then \
+ open -a Docker; \
+ else \
+ sudo systemctl start docker; \
+ fi; \
+ echo "Waiting for Docker to fully initialize..."; \
+ until docker info >/dev/null 2>&1; do sleep 1; done; \
+ echo "Docker is ready!"; \
+ fi
+
.PHONY: checkvenv
-checkvenv:
+checkvenv: startdocker
# raises error if environment is not active
ifeq ("$(VIRTUAL_ENV)","")
@echo "Venv is not activated!"
@@ -119,7 +135,7 @@ test: clean-local-docker-output-dir
docker compose -f docker-compose.yml -f docker-compose.api-test.yml -f docker-compose.override.yml up test-runner
.PHONY: unit-tests
-unit-tests:
+unit-tests: startdocker
docker compose -f docker-compose.yml -f docker-compose.api-test.yml -f docker-compose.override.yml up test-runner
.PHONY: e2e-tests
@@ -133,7 +149,7 @@ e2e-docx-tests: clean-local-docker-output-dir
.PHONY: frontend-tests
-frontend-tests:
+frontend-tests: startdocker
# NOTE If we are experiencing some issues with the docker
# compose running of frontend tests, we can still use the
# non-Dockerized approach successfully. Doing so requires that
@@ -184,7 +200,7 @@ clean-mypyc-artifacts:
find . ! -path .venv -type f -name "*.c" -exec rm -- {} +
.PHONY: prune-docker-images-volumes
-prune-docker-images-volumes:
+prune-docker-images-volumes: checkvenv
docker system prune --volumes
# https://radon.readthedocs.io/en/latest/commandline.html
@@ -301,10 +317,10 @@ local-run-celery:
.PHONY: local-run-flower
local-run-flower:
- celery --broker=redis:// --result-backend=redis:// flower
-# This is one to run after running local-e2e-tests or any tests which
-# has yielded HTML and PDFs that need to be checked for linking
-# correctness.
+ celery --broker=redis:// --result-backend=redis:// flower
+
+# Run after running local-e2e-tests or any tests which has yielded HTML
+# and PDFs that need to be checked for linking correctness.
.PHONY: local-check-anchor-links
local-check-anchor-links: checkvenv
python tests/e2e/test_anchor_linking.py
diff --git a/backend/doc/config.py b/backend/doc/config.py
index f2bf1cfc..10294c59 100755
--- a/backend/doc/config.py
+++ b/backend/doc/config.py
@@ -20,6 +20,8 @@ class Settings(BaseSettings):
DATA_API_URL: HttpUrl
+ LANGUAGES_WHERE_NON_ULB_PREFERRED: Sequence[str] = ["fr"]
+
# This is only used to see if a lang_code is in the collection
# otherwise it is a heart language. Eventually the graphql data api may
# provide gateway/heart boolean value.
@@ -177,7 +179,11 @@ class Settings(BaseSettings):
BOOK_NAME_FMT_STR: str = "{}
"
RESOURCE_TYPE_NAME_FMT_STR: str = "{}
"
HR: str = "
"
- TW_WORD_LIST_VERTICAL: bool = False
+ BIEL_TW_RESOURCE_URL_FMT_STR: str = (
+ "{}"
+ )
+ TW_RESOURCE_URL_FMT_STR: str = "{}"
+ LINK_RATHER_THAN_INCLUDE_TW_DEFINITIONS: bool = True
DOWNLOAD_ASSETS: bool = False # If true then download assets, else clone assets
diff --git a/backend/doc/domain/assembly_strategies/assemble_by_book.py b/backend/doc/domain/assembly_strategies/assemble_by_book.py
index f0c03c50..824670ce 100755
--- a/backend/doc/domain/assembly_strategies/assemble_by_book.py
+++ b/backend/doc/domain/assembly_strategies/assemble_by_book.py
@@ -24,7 +24,7 @@
TWBook,
USFMBook,
)
-from doc.domain.parsing import handle_split_chapter_into_verses
+from doc.domain.parsing import split_chapter_into_verses_with_formatting
from doc.reviewers_guide.model import RGBook
logger = settings.logger(__name__)
@@ -475,7 +475,7 @@ def assemble_usfm_by_verse_book_at_a_time(
chapter_num,
chapter,
) in usfm_book.chapters.items():
- chapter.verses = handle_split_chapter_into_verses(usfm_book, chapter)
+ chapter.verses = split_chapter_into_verses_with_formatting(chapter)
chapter_intros = get_chapter_intros(
tn_book,
tnc_book,
@@ -531,8 +531,10 @@ def assemble_usfm_by_verse_book_at_a_time(
# ulb, f10, show the second USFM content here
if usfm_book2:
usfm_book2_chapter = usfm_book2.chapters[chapter_num]
- usfm_book2_chapter.verses = handle_split_chapter_into_verses(
- usfm_book2, usfm_book2_chapter
+ usfm_book2_chapter.verses = (
+ split_chapter_into_verses_with_formatting(
+ usfm_book2_chapter
+ )
)
if (
usfm_book2_chapter.verses
diff --git a/backend/doc/domain/assembly_strategies/assemble_by_chapter.py b/backend/doc/domain/assembly_strategies/assemble_by_chapter.py
index e141fb4d..29ff87d1 100644
--- a/backend/doc/domain/assembly_strategies/assemble_by_chapter.py
+++ b/backend/doc/domain/assembly_strategies/assemble_by_chapter.py
@@ -37,10 +37,9 @@
TWBook,
USFMBook,
)
-from doc.domain.parsing import handle_split_chapter_into_verses
+from doc.domain.parsing import split_chapter_into_verses_with_formatting
from doc.reviewers_guide.model import RGBook
-
logger = settings.logger(__name__)
@@ -625,13 +624,13 @@ def assemble_usfm_by_verse_chapter_at_a_time(
primary_verses = None
secondary_verses = None
if usfm_book and usfm_chapter:
- usfm_chapter.verses = handle_split_chapter_into_verses(
- usfm_book, usfm_chapter
+ usfm_chapter.verses = split_chapter_into_verses_with_formatting(
+ usfm_chapter
)
primary_verses = usfm_chapter.verses
if usfm_book2 and usfm_book2_chapter:
- usfm_book2_chapter.verses = handle_split_chapter_into_verses(
- usfm_book2, usfm_book2_chapter
+ usfm_book2_chapter.verses = split_chapter_into_verses_with_formatting(
+ usfm_book2_chapter
)
secondary_verses = usfm_book2_chapter.verses
if usfm_book and primary_verses:
diff --git a/backend/doc/domain/document_generator.py b/backend/doc/domain/document_generator.py
index 7d223e58..0b5c488b 100755
--- a/backend/doc/domain/document_generator.py
+++ b/backend/doc/domain/document_generator.py
@@ -51,6 +51,7 @@
from doc.utils.docx_util import (
add_internal_docx_links,
generate_docx_toc,
+ override_and_clean_hyperlinks,
style_superscripts,
)
from doc.utils.file_utils import (
@@ -558,6 +559,7 @@ def assemble_content(
tw_books: Sequence[TWBook],
bc_books: Sequence[BCBook],
rg_books: Sequence[RGBook],
+ link_rather_than_include_tw_definitions: bool = settings.LINK_RATHER_THAN_INCLUDE_TW_DEFINITIONS,
) -> list[DocumentPart]:
"""
Assemble and return the content from all requested resources according to the
@@ -651,7 +653,11 @@ def assemble_content(
)
t1 = time.time()
logger.info("Time for interleaving document: %s", t1 - t0)
- if tw_books and not document_request.layout_for_print:
+ if (
+ tw_books
+ and not link_rather_than_include_tw_definitions
+ and not document_request.layout_for_print
+ ):
t0 = time.time()
# Add the translation words definition section for each language requested.
unique_tw_books = filter_unique_by_lang_code(tw_books)
@@ -778,6 +784,15 @@ def compose_docx_document(
if part.add_page_break:
add_page_break(doc)
style_superscripts(doc, lift_half_points=2, color=None)
+ # html4doc defeats normal use of hyperlink inline styling in Word
+ # via a customized (otherwise standard) Hyperlink style, but
+ # this handles it by doing a pass over the document after the fact
+ # and forcing hyperlinks to render using a specific style we
+ # created to match the PO's desired look.
+ # PlainHyperlinkChar is a character style we created in
+ # template.docx, but its actual style ID is PlainHyperlinkChar0
+ # under the hood.
+ override_and_clean_hyperlinks(doc, "PlainHyperlinkChar0")
t1 = time.time()
logger.info("Time for converting HTML to Docx: %.2f seconds", t1 - t0)
return doc
diff --git a/backend/doc/domain/parsing.py b/backend/doc/domain/parsing.py
index 2423f68c..08c0c255 100644
--- a/backend/doc/domain/parsing.py
+++ b/backend/doc/domain/parsing.py
@@ -2,6 +2,7 @@
This module provides an API for parsing content.
"""
+from bs4 import BeautifulSoup, NavigableString
from re import (
compile,
escape,
@@ -276,14 +277,19 @@ def split_usfm_by_chapters(
chapters = re_split(chapter_regex, usfm_text)
frontmatter = chapters.pop(0).strip()
+ defective_lang_codes = {resource[0] for resource in resources_with_usfm_defects}
+
def needs_fixing() -> bool:
"""
- Determine if a chapter needs fixing based on configuration.
+ Determine whether this resource should be checked for known USFM defects.
+
+ If CHECK_ALL_BOOKS_FOR_LANGUAGE is enabled, then the presence of any
+ known defective resource for a language causes all books for that
+ language to be checked for similar defects. Otherwise, only explicitly
+ listed resource tuples are checked.
"""
if check_all_books_for_language:
- return lang_code in [
- resource[0] for resource in resources_with_usfm_defects
- ]
+ return lang_code in defective_lang_codes
return (
lang_code,
resource_type,
@@ -381,7 +387,6 @@ def remove_null_bytes_and_control_characters(html_content: Optional[str]) -> str
def extract_usfm_frontmatter(frontmatter: str) -> dict[str, str]:
- # Define the regex patterns to match \h, \mt, and \toc
patterns = {
"h": r"\\h\s+(.*?)(?=\s+\\|\n|$)",
"mt": r"\\mt\s+(.*?)(?=\s+\\|\n|$)",
@@ -396,37 +401,63 @@ def extract_usfm_frontmatter(frontmatter: str) -> dict[str, str]:
return extracted_data
-def maybe_localized_book_name(frontmatter: str) -> str:
- r"""
- Rule for obtaining localized book name:
-
- In USFM:
+# Global defaults for fallback/reference
+DEFAULT_BOOK_NAME_LOOKUP_ORDER = ["h", "mt", "toc1", "toc2"]
- 1. Look to see if the \h marker is present — if so, use that value.
- 2. Else look to see if the \mt1 marker is present — if so, use that value.
- 3. Else look to see if the \toc1 marker is present - if so, use that value.
- 4. Else look to see if the \toc2 marker is present - if so, use that value.
+SPECIALIZED_BOOK_NAME_LOOKUP_MAP: dict[tuple[str, str], list[str]] = {
+ ("fr", "f10"): ["toc2"],
+}
- Outside USFM:
+# Define combinations that should skip normalization
+SKIP_NORMALIZATION_SET: set[tuple[str, str]] = {
+ ("fr", "f10"), # Skip normalization for French f10
+}
- 5. Else use the book name from the source language if available.
- 6. Otherwise use the English book name.
- Steps 5 and 6 happen outside this function.
+def maybe_localized_book_name(
+ frontmatter: str,
+ language: str,
+ resource_type: str,
+ default_book_name_lookup_order: list[str] = DEFAULT_BOOK_NAME_LOOKUP_ORDER,
+ specialized_book_name_lookup_map: dict[
+ tuple[str, str], list[str]
+ ] = SPECIALIZED_BOOK_NAME_LOOKUP_MAP,
+ skip_normalization_set: set[tuple[str, str]] = SKIP_NORMALIZATION_SET,
+) -> str:
+ """
+ Rule for obtaining localized book name based on language and resource type.
+ Falls back to empirical default sequence if no specialization exists.
+ Allows skipping normalization for specific language/resource combinations.
"""
frontmatter_data = extract_usfm_frontmatter(frontmatter)
- localized_book_name = (
- frontmatter_data.get("h")
- or frontmatter_data.get("mt")
- or frontmatter_data.get("mt1")
- or frontmatter_data.get("toc1")
- or frontmatter_data.get("toc2")
- or ""
+ # Normalize inputs for lookup consistency
+ lang_key = language.lower()
+ res_key = resource_type.lower()
+ lookup_key = (lang_key, res_key)
+ # 1. Determine the marker lookup order (Specialized vs Default)
+ marker_order = specialized_book_name_lookup_map.get(
+ lookup_key, default_book_name_lookup_order
+ )
+ # 2. Iterate through the preferred markers and grab the first one that exists
+ localized_book_name = ""
+ for marker in marker_order:
+ value = frontmatter_data.get(marker)
+ if value:
+ localized_book_name = value
+ break
+ logger.debug(
+ "Using marker order %s for (%s, %s). Found: %s",
+ marker_order,
+ language,
+ resource_type,
+ localized_book_name,
)
- logger.debug("localized_book_name: %s", localized_book_name)
+ # 3. Normalize and clean up if a name was found and not explicitly skipped
if localized_book_name:
- localized_book_name = normalize_localized_book_name(localized_book_name)
- logger.debug("normalized localized_book_name: %s", localized_book_name)
+ if lookup_key in skip_normalization_set:
+ logger.debug("Skipping normalization for %s", lookup_key)
+ else:
+ localized_book_name = normalize_localized_book_name(localized_book_name)
return localized_book_name
@@ -527,9 +558,7 @@ def usfm_book_content(
cleaned_chapter_html_content_ = remove_null_bytes_and_control_characters(
chapter_html_content
)
- cleaned_chapter_html_content = remove_unwanted_elements(
- cleaned_chapter_html_content_
- )
+ cleaned_chapter_html_content = clean_content_html(cleaned_chapter_html_content_)
usfm_chapters[chapter_num] = USFMChapter(
content=(
cleaned_chapter_html_content if cleaned_chapter_html_content else ""
@@ -544,7 +573,7 @@ def usfm_book_content(
national_book_name=(
localized_book_name
if localized_book_name
- else BOOK_NAMES[resource_lookup_dto.book_code]
+ else book_names[resource_lookup_dto.book_code]
),
resource_type_name=resource_lookup_dto.resource_type_name,
chapters=usfm_chapters if usfm_chapters else {},
@@ -558,7 +587,11 @@ def get_localized_book_name(
resource_lookup_dto: ResourceLookupDto,
usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
) -> str:
- localized_book_name = maybe_localized_book_name(frontmatter)
+ localized_book_name = maybe_localized_book_name(
+ frontmatter,
+ resource_lookup_dto.lang_code,
+ resource_lookup_dto.resource_type,
+ )
if not localized_book_name:
book_codes_and_names_from_manifest_ = book_codes_and_names_from_manifest(
resource_dir
@@ -1408,172 +1441,110 @@ def lookup_verse_text(usfm_book: USFMBook, chapter_num: int, verse_ref: str) ->
return verse
-# Used by STET and PASSAGES apps
-def split_chapter_into_verses(chapter: USFMChapter) -> dict[str, str]:
- # Sample HTML content with multiple verse elements
- # html_content = '''
- #
- # 19
- # For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
- #
- #
- #
- #
- # 20
- # I have been crucified with Christ and I no longer live, but Christ lives in me. The life I now live in the body, I live by faith in the Son of God, who loved me and gave himself for me.
- #
- #
- #
- # '''
- verse_dict = {}
- # Find all verse spans
- verse_spans = findall(r'(.*?)', chapter.content, DOTALL)
- for verse_span in verse_spans:
- # Extract the verse number from the versemarker
- verse_number = search(r'(\d+)', verse_span)
- if verse_number:
- verse_number_ = verse_number.group(1)
- # Remove versemarker
- verse_text = sub(r'.*?', "", verse_span)
- # Remove footnotes numbers
- verse_text = sub(r'.*?', "", verse_text)
- # Fix spacing issue when div class="poetry-*" type divs
- # are used, e.g., yielding 'heartsas' for Hebrews 3:8
- verse_text = sub(
- r'(.*?)
',
- r" \1",
- verse_text,
- )
- # Add to the dictionary with verse number as the key and verse text as the value
- verse_dict[verse_number_] = verse_text
- return verse_dict
-
-
-def handle_split_chapter_into_verses(
- usfm_book: USFMBook,
- usfm_chapter: USFMChapter,
- resource_type_codes_and_names: Mapping[
- str, str
- ] = settings.RESOURCE_TYPE_CODES_AND_NAMES,
-) -> dict[VerseRef, str]:
- if (
- usfm_book.lang_code == "fr"
- and usfm_book.resource_type_name == resource_type_codes_and_names["f10"]
- ):
- return split_chapter_into_verses_with_formatting_for_f10(usfm_chapter)
- else:
- return split_chapter_into_verses_with_formatting(usfm_chapter)
-
-
def split_chapter_into_verses_with_formatting(
chapter: USFMChapter,
- empty_paragraph: str = "",
- sectionhead5_element: str = '',
) -> dict[VerseRef, str]:
"""
- Given a USFMChapter instance, return the same instance with its
- verses attribute set to a dictionary where the key is the verse
- number and the value is the verse HTML.
+ Parse chapter.content as HTML, extract each ,
+ unwrap elements (preserving their text),
+ and return a dict mapping verse number -> cleaned HTML fragment for that verse.
Sample HTML content with multiple verse elements:
>>> html_content = '''
...
- ... 19
- ... For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
- ...
- ...
+ ... 1
+ ... Généalogie
+ ...
+ ... de
+ ... Jésus
+ ... -
+ ... Christ
+ ... ,
+ ... fils
+ ... de
+ ... David
+ ... ,
+ ... fils
+ ... d'
+ ... Abraham
+ ... .
+ ...
...
...
- ... 20
- ... I have been crucified with Christ and I no longer live, but Christ lives in me. The life I now live in the body, I live by faith in the Son of God, who loved me and gave himself for me.
- ...
- ...
+ ... 2
+ ... Abraham
+ ...
+ ... engendra
+ ...
+ ... Isaac
+ ... ;
+ ...
+ ...
+ ... Isaac
+ ...
+ ... engendra
+ ...
+ ... Jacob
+ ... ;
+ ...
+ ...
+ ... Jacob
+ ...
+ ... engendra
+ ...
+ ... Juda
+ ...
+ ... et
+ ...
+ ... ses
+ ...
+ ... frères
+ ... ;
...
... '''
>>> from doc.domain.parsing import split_chapter_into_verses_with_formatting
>>> chapter = USFMChapter(content=html_content, verses=None)
>>> chapter.verses = split_chapter_into_verses_with_formatting(chapter)
- >>> print(chapter.verses["19"])
- 19
- For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
-
-
- """
- # TODO What to do about footnote targets? Perhaps have the value be a
- # tuple with first element of the verse HTML (which includes the
- # footnote callers) and the second element the target footnotes HTML?
- verse_dict = {}
- # Find all verse spans
- verse_spans = findall(r'(.*?)', chapter.content, DOTALL)
- for verse_span in verse_spans:
- # Extract the verse number from the versemarker
- verse_number = search(r'(\d+)', verse_span)
- if verse_number:
- verse_number_ = verse_number.group(1)
- # Add to the dictionary with verse number as the key and verse text as the value
- verse_dict[verse_number_] = (
- verse_span.strip()
- .replace(empty_paragraph, "")
- .replace(sectionhead5_element, "")
- )
- return verse_dict
-
-
-def split_chapter_into_verses_with_formatting_for_f10(
- chapter: USFMChapter,
- empty_paragraph: str = "",
- sectionhead5_element: str = '',
-) -> dict[str, str]:
- """
- Parse chapter.content as HTML, extract each ,
- unwrap elements (preserving their text),
- and return a dict mapping verse number -> cleaned HTML fragment for that verse.
+ >>> print(chapter.verses["1"])
+ Généalogie de Jésus-Christ, fils de David, fils d'Abraham.
"""
soup = BeautifulSoup(chapter.content, "html.parser")
- verse_dict: dict[str, str] = {}
- # find all verse spans (parser handles nesting correctly)
+ verse_dict: dict[VerseRef, str] = {}
for verse_span in soup.find_all("span", class_="verse"):
- # find the verse number from NN
sup = verse_span.find("sup", class_="versemarker")
if not sup or not sup.string:
continue
verse_number = sup.string.strip()
- # unwrap all word-entry spans: replace X
- # with X (preserving whitespace/punctuation)
+ # Remove the versemarker number sup
+ sup.decompose()
+ # fr f10 uses word-entry tags
for we in verse_span.find_all("span", class_="word-entry"):
we.unwrap()
- # Option: normalize whitespace (optional)
- # If you want to preserve original spacing/punctuation exactly, skip this.
- # cleaned_html = "".join(str(c) for c in verse_span.contents)
- cleaned_html = str(verse_span)
- # Fix spacing issues introduced by inner spans
- cleaned_html = sub(
- r"\s+([,;:.!?])", r"\1", cleaned_html
- ) # remove space before punctuation
- cleaned_html = sub(r"\s+'", "'", cleaned_html) # remove space before apostrophe
- cleaned_html = sub(r"'\s+", "'", cleaned_html) # remove space after apostrophe
- cleaned_html = sub(
- r"\s*-\s*", "-", cleaned_html
- ) # normalize spaces around hyphens
- cleaned_html = sub(r"\s{2,}", " ", cleaned_html) # collapse double spaces
- cleaned_html = cleaned_html.strip()
- # if you want plain text instead, use: cleaned_text = verse_span.get_text(" ", strip=True)
- # store cleaned HTML fragment (still contains etc.)
- verse_dict[verse_number] = (
- cleaned_html.strip()
- .replace(empty_paragraph, "")
- .replace(sectionhead5_element, "")
- )
+ cleaned_html = clean_content_html(str(verse_span))
+ verse_dict[verse_number] = cleaned_html
return verse_dict
+def clean_content_html(raw_content: str) -> str:
+ soup = BeautifulSoup(raw_content, "html.parser")
+ cleaned_html = str(soup)
+ cleaned_html = sub(r"\s+([,;:.!?])", r"\1", cleaned_html)
+ cleaned_html = sub(r"\s+'", "'", cleaned_html)
+ cleaned_html = sub(r"'\s+", "'", cleaned_html)
+ cleaned_html = sub(r"\s*-\s*", "-", cleaned_html)
+ cleaned_html = sub(r"\s{2,}", " ", cleaned_html).strip()
+ return cleaned_html
+
+
if __name__ == "__main__":
# To run the doctests in this module, in the root of the project do:
- # python backend/document/domain/resource_lookup.py
+ # PYTHONPATH=backend python backend/doc/domain/parsing.py
# or
- # python backend/document/domain/resource_lookup.py -v
+ # PYTHONPATH=backend python backend/doc/domain/parsing.py -v
+ # These doctests are not collected by pytest: pyproject.toml sets
+ # testpaths = ["tests"] with no doctest collection, so run them by hand.
# See https://docs.python.org/3/library/doctest.html
# for more details.
import doctest
diff --git a/backend/doc/domain/resource_lookup.py b/backend/doc/domain/resource_lookup.py
index f3da9d4c..e0de8ec0 100644
--- a/backend/doc/domain/resource_lookup.py
+++ b/backend/doc/domain/resource_lookup.py
@@ -55,15 +55,6 @@
)
-# List of languages which do not have USFM available for any books. We use this
-# to filter these out of STET's list of source and target
-# languages so that the user doesn't have the frustrating experience of
-# selecting a language which might have non-USFM resources available but
-# not USFM so that when their resulting doc is generated no scripture is
-# present. It makes it seem like a bug in STET and is bad UX.
-LANG_CODES_WITH_NO_USFM: frozenset[str] = frozenset(["ru"])
-
-
@cached(fetch_source_data_cache)
def fetch_source_data(
data_api_url: HttpUrl = settings.DATA_API_URL,
@@ -77,7 +68,7 @@ def fetch_source_data(
>>> ();result = resource_lookup.fetch_source_data();() # doctest: +ELLIPSIS
(...)
>>> result.git_repo[0]
- RepoEntry(repo_url=HttpUrl('https://content.bibletranslationtools.org/0success/cli_1jn_text_reg'), content=Content(resource_type='reg', language=Language(english_name='Chakali', ietf_code='cli', national_name='Chakali', direction=)))
+ RepoEntry(repo_url=HttpUrl('https://content.bibletranslationtools.org/0success/cli_1co_text_reg'), content=Content(resource_type='reg', language=Language(english_name='Chakali', ietf_code='cli', national_name='Chakali', direction=)))
"""
graphql_query = """
query MyQuery {
@@ -147,7 +138,7 @@ def lang_codes_and_names(
('abz', 'Abui', False)
>>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names() if not lang_code_and_name[2]]
>>> sorted(heart_lang_codes)[0]
- 'aao'
+ 'aac'
"""
data = fetch_source_data()
values = []
@@ -175,46 +166,6 @@ def lang_codes_and_names(
return sorted(unique_values, key=lambda value: value[1])
-def lang_codes_and_names_having_usfm(
- lang_code_filter_list: frozenset[str] = LANG_CODES_WITH_NO_USFM,
- gateway_languages: frozenset[str] = settings.GATEWAY_LANGUAGES,
-) -> Sequence[tuple[str, str, bool]]:
- """
- >>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.lang_codes_and_names_having_usfm();() # doctest: +ELLIPSIS
- (...)
- >>> result[0]
- ('abz', 'Abui', False)
- >>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm() if not lang_code_and_name[2]]
- >>> sorted(heart_lang_codes)[0]
- 'aao'
- """
- data = fetch_source_data()
- values = []
- if data is None or not data.git_repo:
- logger.info("Data API is down or no git_repo found!")
- return []
- try:
- for repo_info in data.git_repo:
- language_info = repo_info.content
- language = language_info.language
- ietf_code = language.ietf_code
- english_name = language.english_name if language.english_name else ""
- localized_name = language.national_name
- is_gateway = ietf_code in gateway_languages
- if ietf_code not in lang_code_filter_list:
- if english_name in localized_name:
- values.append((ietf_code, localized_name, is_gateway))
- else:
- values.append(
- (ietf_code, f"{localized_name} ({english_name})", is_gateway)
- )
- except Exception:
- logger.exception("Failed due to the following exception.")
- unique_values = unique_tuples(values)
- return sorted(unique_values, key=lambda value: value[1])
-
-
def repos_to_clone(
lang_code: str,
augmented_repos_info: list[RepoEntry],
@@ -289,22 +240,6 @@ def get_resource_types(
]
elif resource_type in usfm_resource_types:
book_assets = find_usfm_files(resource_filepath)
- elif resource_type == "rg":
- between_texts, bible_reference_strs = find_bible_references(
- join(en_rg, docx_file_path)
- )
- bible_references = [
- parse_bible_reference(bible_reference)
- for bible_reference in bible_reference_strs
- ]
- book_codes_ = {
- bible_reference.book_code
- for bible_reference in bible_references
- if bible_reference
- }
- book_assets = [
- book_code for book_code in book_codes if book_code in book_codes_
- ]
if book_assets or resource_type == "tw":
resource_types.append(
(
@@ -623,7 +558,7 @@ def shared_book_codes(lang0_code: str, lang1_code: str) -> Sequence[tuple[str, s
>>> ();data = resource_lookup.book_codes_for_lang("pt-br");() # doctest: +ELLIPSIS
(...)
>>> list(data)
- [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levítico'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
+ [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levíticos'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares de salomão'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
>>> ();data = resource_lookup.book_codes_for_lang("es-419");() # doctest: +ELLIPSIS
(...)
>>> list(data)
@@ -631,7 +566,7 @@ def shared_book_codes(lang0_code: str, lang1_code: str) -> Sequence[tuple[str, s
>>> ();data = resource_lookup.shared_book_codes("pt-br", "es-419");() # doctest: +ELLIPSIS
(...)
>>> list(data)
- [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levítico'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
+ [('gen', 'Gênesis'), ('exo', 'Êxodo'), ('lev', 'Levíticos'), ('num', 'Números'), ('deu', 'Deuteronômio'), ('jos', 'Josué'), ('jdg', 'Juízes'), ('rut', 'Rute'), ('1sa', '1 Samuel'), ('2sa', '2 Samuel'), ('1ki', '1 Reis'), ('2ki', '2 Reis'), ('1ch', '1 Crônicas'), ('2ch', '2 Crônicas'), ('ezr', 'Esdras'), ('neh', 'Neemias'), ('est', 'Ester'), ('job', 'Jó'), ('psa', 'Salmos'), ('pro', 'Provérbios'), ('ecc', 'Eclesiastes'), ('sng', 'Cantares de salomão'), ('isa', 'Isaías'), ('jer', 'Jeremias'), ('lam', 'Lamentações'), ('ezk', 'Ezequiel'), ('dan', 'Daniel'), ('hos', 'Oseias'), ('jol', 'Joel'), ('amo', 'Amós'), ('oba', 'Obadias'), ('jon', 'Jonas'), ('mic', 'Miqueias'), ('nam', 'Naum'), ('hab', 'Habacuque'), ('zep', 'Sofonias'), ('hag', 'Ageu'), ('zec', 'Zacarias'), ('mal', 'Malaquias'), ('mat', 'Mateus'), ('mrk', 'Marcos'), ('luk', 'Lucas'), ('jhn', 'João'), ('act', 'Atos'), ('rom', 'Romanos'), ('1co', '1 Coríntios'), ('2co', '2 Coríntios'), ('gal', 'Gálatas'), ('eph', 'Efésios'), ('php', 'Filipenses'), ('col', 'Colossenses'), ('1th', '1 Tessalonicenses'), ('2th', '2 Tessalonicenses'), ('1ti', '1 Timóteo'), ('2ti', '2 Timóteo'), ('tit', 'Tito'), ('phm', 'Filemom'), ('heb', 'Hebreus'), ('jas', 'Tiago'), ('1pe', '1 Pedro'), ('2pe', '2 Pedro'), ('1jn', '1 João'), ('2jn', '2 João'), ('3jn', '3 João'), ('jud', 'Judas'), ('rev', 'Apocalipse')]
"""
lang0_book_codes = book_codes_for_lang(lang0_code)
@@ -963,7 +898,9 @@ def get_book_names_from_usfm_metadata(
frontmatter, _, _ = split_usfm_by_chapters(
lang_code, resource_type, book_code, usfm
)
- localized_book_name = maybe_localized_book_name(frontmatter)
+ localized_book_name = maybe_localized_book_name(
+ frontmatter, lang_code, resource_type
+ )
# localized_book_name = maybe_correct_book_name(lang_code, localized_book_name)
book_codes_and_names_localized[book_code] = localized_book_name
logger.debug("book_codes_and_names_localized: %s", book_codes_and_names_localized)
@@ -1213,14 +1150,14 @@ def nt_survey_rg_passages(
resource_dir: str = settings.EN_RG_DIR,
) -> list[BibleReference]:
"""
- Returns the list of all NT RG passages from the docx_file_path, but with
+ Returns the list of all NT RG passages from the docx_file_path, but with
book names localized for language chosen.
- >>> from doc.domain import resource_lookup
- >>> ();rg_books = resource_lookup.nt_survey_rg_passages() ;() # doctest: +ELLIPSIS
- (...)
- >>> rg_books[0]
- BibleReference(book_code='mat', book_name='Matthew', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
+ >>> from doc.domain import resource_lookup
+ >>> ();rg_books = resource_lookup.nt_survey_rg_passages() ;() # doctest: +ELLIPSIS
+ (...)
+ >>> rg_books[0]
+ BibleReference(lang_code='en', book_code='mat', book_name='Matthew', start_chapter=2, start_chapter_verse_ref='1-12', end_chapter=None, end_chapter_verse_ref=None)
"""
path = join(resource_dir, docx_file_path)
rg_books = get_rg_books(
@@ -1263,7 +1200,7 @@ def ot_survey_rg1_passages(
>>> ();rg_books = resource_lookup.ot_survey_rg1_passages();() # doctest: +ELLIPSIS
(...)
>>> rg_books[0]
- BibleReference(book_code='gen', book_name='Genesis', start_chapter=1, start_chapter_verse_ref='1', end_chapter=2, end_chapter_verse_ref='3')
+ BibleReference(lang_code='en', book_code='gen', book_name='Genesis', start_chapter=1, start_chapter_verse_ref='1', end_chapter=2, end_chapter_verse_ref='3')
"""
path = join(resource_dir, docx_file_path)
rg_books = get_rg_books(
@@ -1306,7 +1243,7 @@ def ot_survey_rg2_passages(
>>> ();rg_books = resource_lookup.ot_survey_rg2_passages();() # doctest: +ELLIPSIS
(...)
>>> rg_books[0]
- BibleReference(book_code='jos', book_name='Joshua', start_chapter=1, start_chapter_verse_ref='1-9', end_chapter=None, end_chapter_verse_ref=None)
+ BibleReference(lang_code='en', book_code='jos', book_name='Joshua', start_chapter=1, start_chapter_verse_ref='1-9', end_chapter=None, end_chapter_verse_ref=None)
"""
path = join(resource_dir, docx_file_path)
rg_books = get_rg_books(
@@ -1349,7 +1286,7 @@ def ot_survey_rg3_passages(
>>> ();rg_books = resource_lookup.ot_survey_rg3_passages();() # doctest: +ELLIPSIS
(...)
>>> rg_books[0]
- BibleReference(book_code='job', book_name='Job', start_chapter=1, start_chapter_verse_ref='6-22', end_chapter=None, end_chapter_verse_ref=None)
+ BibleReference(lang_code='en', book_code='job', book_name='Job', start_chapter=1, start_chapter_verse_ref='6-22', end_chapter=None, end_chapter_verse_ref=None)
"""
path = join(resource_dir, docx_file_path)
rg_books = get_rg_books(
@@ -1392,7 +1329,7 @@ def ot_survey_rg4_passages(
>>> ();rg_books = resource_lookup.ot_survey_rg4_passages();() # doctest: +ELLIPSIS
(...)
>>> rg_books[0]
- BibleReference(book_code='isa', book_name='Isaiah', start_chapter=1, start_chapter_verse_ref='1-9', end_chapter=None, end_chapter_verse_ref=None)
+ BibleReference(lang_code='en', book_code='isa', book_name='Isaiah', start_chapter=1, start_chapter_verse_ref='1-9', end_chapter=None, end_chapter_verse_ref=None)
"""
path = join(resource_dir, docx_file_path)
rg_books = get_rg_books(
diff --git a/backend/doc/domain/usfm_error_detection_and_fixes.py b/backend/doc/domain/usfm_error_detection_and_fixes.py
index f97268b8..66ddf8c9 100644
--- a/backend/doc/domain/usfm_error_detection_and_fixes.py
+++ b/backend/doc/domain/usfm_error_detection_and_fixes.py
@@ -1,18 +1,26 @@
-import re
-from typing import Sequence
+from collections.abc import Callable
+from re import Match, Pattern, compile, findall, finditer, sub
+from typing import Mapping, Sequence, TypeAlias
from doc.config import settings
logger = settings.logger(__name__)
-# Resources known to have USFM defects found through automatic
-# randomized testing and subsequent manual investigation. As an aside:
-# When we find one defective USFM resource for a language then the
-# language might have others. In keeping with that fact one can set
-# the value of settings.CHECK_ALL_BOOKS_FOR_LANGUAGE in .env file.
-# This list is also used in tests and in that context
-# settings.CHECK_ALL_BOOKS_FOR_LANGUAGE is not checked but each
-# resource listed below is tested.
+
+Detector: TypeAlias = Callable[[str], Match[str] | None]
+Fixer: TypeAlias = Callable[[str], str]
+ContextFormatter: TypeAlias = Callable[[str, Match[str]], str]
+Rule: TypeAlias = tuple[str, Detector, Fixer, ContextFormatter | None]
+
+
+# Resources known to have USFM defects found through randomized testing
+# and subsequent manual investigation. Some entries were added after
+# confirming that the detected issue was fixable in a reasonable way.
+#
+# This list is also used to broaden checking: when
+# settings.CHECK_ALL_BOOKS_FOR_LANGUAGE is enabled, if one resource for a
+# language is known to have a defect, then all books for that language are
+# checked for similar defects.
RESOURCES_WITH_USFM_DEFECTS: Sequence[tuple[str, str, str]] = [
("aaz-x-amarasibarat", "reg", "2pe"),
("abu", "reg", "php"),
@@ -21,7 +29,6 @@
("adn", "reg", "mat"),
("agd-x-namel", "reg", "2th"),
("ahm", "reg", "php"),
- ("ahm", "reg", "php"),
("ajg-x-adjtalagbe", "reg", "mat"),
("aoa", "reg", "col"),
("aoa", "reg", "luk"),
@@ -52,7 +59,6 @@
("blo", "reg", "php"),
("blo", "reg", "heb"),
("blo", "reg", "1jn"),
- ("blo", "reg", "col"),
("bne", "reg", "gal"),
("bof", "reg", "mat"),
("bou", "reg", "gen"),
@@ -269,8 +275,7 @@
]
-# List of regex patterns to detect issues before applying corrections
-pattern_matchers = {
+PATTERN_MATCHERS = {
"remove_null_bytes_and_control_characters": r"[\x00-\x1F]+",
"fix_dot_after_verse_number": r"(\\v\s*\d+)\s*\.\s*(\S)",
"fix_verse_marker_without_v": r"\\(\d+)\s*\.?\s*(\S+)",
@@ -290,12 +295,57 @@
"replace_cc_with_c": r"(\\c \d+)\s+(\\c \d+)",
}
-compiled_patterns = {
- key: re.compile(pattern) for key, pattern in pattern_matchers.items()
-}
+COMPILED_PATTERNS = {key: compile(pattern) for key, pattern in PATTERN_MATCHERS.items()}
+
+
+def make_detector(
+ pattern_name: str,
+ compiled_patterns: Mapping[str, Pattern[str]] = COMPILED_PATTERNS,
+) -> Detector:
+ def detector(content: str) -> Match[str] | None:
+ return compiled_patterns[pattern_name].search(content)
+
+ return detector
+
+def default_context_formatter(content: str, match: Match[str]) -> str:
+ return content[max(0, match.start() - 5) : min(len(content), match.end() + 5)]
-def remove_null_bytes_and_control_characters(content: str) -> str:
+
+def log_detection(
+ rule_name: str,
+ match: Match[str],
+ content: str,
+ lang_code: str,
+ resource_type: str,
+ book_code: str,
+ context_formatter: ContextFormatter | None = None,
+) -> None:
+ formatter = context_formatter or default_context_formatter
+ context = formatter(content, match)
+ logger.debug(
+ "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s",
+ rule_name,
+ match.group(),
+ context,
+ lang_code,
+ resource_type,
+ book_code,
+ )
+
+
+def make_rule(
+ rule_name: str,
+ fixer: Fixer,
+ context_formatter: ContextFormatter | None = None,
+) -> Rule:
+ return (rule_name, make_detector(rule_name), fixer, context_formatter)
+
+
+def remove_null_bytes_and_control_characters(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
"""
Remove any NULL bytes and all control characters.
@@ -303,28 +353,33 @@ def remove_null_bytes_and_control_characters(content: str) -> str:
USFM. We strip those out as well as the possibility of ASCII NULL
bytes.
"""
- return re.sub(
+ return sub(
pattern_matchers["remove_null_bytes_and_control_characters"], "", content
)
-def fix_dot_after_verse_number(content: str) -> str:
- return re.sub(pattern_matchers["fix_dot_after_verse_number"], r"\1 \2", content)
+def fix_dot_after_verse_number(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
+ return sub(pattern_matchers["fix_dot_after_verse_number"], r"\1 \2", content)
-def fix_verse_marker_without_v(content: str) -> str:
- return re.sub(pattern_matchers["fix_verse_marker_without_v"], r"\\v \1 \2", content)
+def fix_verse_marker_without_v(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
+ return sub(pattern_matchers["fix_verse_marker_without_v"], r"\\v \1 \2", content)
-def fix_missing_space_before_number(content: str) -> str:
+def fix_missing_space_before_number(
+ content: str,
+ compiled_patterns: Mapping[str, Pattern[str]] = COMPILED_PATTERNS,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
if match := compiled_patterns["fix_missing_space_before_number"].search(content):
- # logger.debug(
- # "match.group(1): %s",
- # match.group(1),
- # )
character_before_number = content[match.start() - 1]
character_before_before_number = content[match.start() - 2]
- # logger.debug("character_before_number: %s", character_before_number)
if (
character_before_number.isdigit()
and match.group(1).isdigit()
@@ -336,15 +391,15 @@ def fix_missing_space_before_number(content: str) -> str:
"Actually, it wasn't missing a space before number after all upon further checking"
)
return content
- else:
- return re.sub(
- pattern_matchers["fix_missing_space_before_number"], r" \1", content
- )
+ return sub(pattern_matchers["fix_missing_space_before_number"], r" \1", content)
return content
-def fix_missing_space_after_number(content: str) -> str:
- matches = re.findall(compiled_patterns["fix_missing_space_after_number"], content)
+def fix_missing_space_after_number(
+ content: str,
+ compiled_patterns: Mapping[str, Pattern[str]] = COMPILED_PATTERNS,
+) -> str:
+ matches = findall(compiled_patterns["fix_missing_space_after_number"], content)
for match in matches:
if not match[2].isdigit() and match[2][0] not in [
":",
@@ -358,17 +413,24 @@ def fix_missing_space_after_number(content: str) -> str:
]: # Skip e.g., '(Zak 13:9)'
replacement = match[1] + " " + match[2]
pattern2 = f"{match[1]}{match[2]}"
- content = re.sub(pattern2, replacement, content)
+ content = sub(pattern2, replacement, content)
return content
-def fix_missing_space_before_verse_marker(content: str) -> str:
- return re.sub(
+def fix_missing_space_before_verse_marker(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
+ return sub(
pattern_matchers["fix_missing_space_before_verse_marker"], r"\1 \2", content
)
-def fix_standalone_verse_numbers(content: str) -> str:
+def fix_standalone_verse_numbers(
+ content: str,
+ compiled_patterns: Mapping[str, Pattern[str]] = COMPILED_PATTERNS,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
# We have to try to determine if a standalone integer should be
# interpreted as a verse with missing verse marker or as a valid piece
# of non-structural content. E.g., in ayn, mat, chapter 1, verse 17, the
@@ -404,30 +466,31 @@ def fix_standalone_verse_numbers(content: str) -> str:
# Extract all standalone numbers (likely verse numbers) from
# content but skip the first part, 6 characters, of content
# which could contain a chapter marker and its value.
- matches = [int(m.group()) for m in re.finditer(r"\b\d+\b", content[7:])]
- # logger.debug("standalone number matches: %s", matches)
+ matches = [int(m.group()) for m in finditer(r"\b\d+\b", content[7:])]
is_ascending = all(
earlier < later for earlier, later in zip(matches, matches[1:])
)
- # logger.debug("is_ascending: %s", is_ascending)
num_matches = len(matches)
if (
- not re.compile(r"""\\v \d+""").search(context_for_standalone_verse)
+ not compile(r"""\\v \d+""").search(context_for_standalone_verse)
and not num_matches >= num_of_occurrences
) or is_ascending: # Check for non-ascending numbers
- return re.sub(
+ return sub(
pattern_matchers["fix_standalone_verse_numbers"],
r"\\v \1",
content,
)
- else:
- logger.info(
- "Actually, we can't be certain it was a standalone verse number after all upon further checking"
- )
+ logger.info(
+ "Actually, we can't be certain it was a standalone verse number after all upon further checking"
+ )
return content
-def fix_standalone_verse_number_and_period(content: str) -> str:
+def fix_standalone_verse_number_and_period(
+ content: str,
+ compiled_patterns: Mapping[str, Pattern[str]] = COMPILED_PATTERNS,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
# E.g., in Russian (ru) some of the USFM exhibits verse markers of the
# form 1. rather than \v 1
if match := compiled_patterns["fix_standalone_verse_number_and_period"].search(
@@ -446,7 +509,7 @@ def fix_standalone_verse_number_and_period(content: str) -> str:
# Extract all standalone number and period (likely verse numbers) from
# content but skip the first part, 6 characters, of content
# which could contain a chapter marker and its value.
- matches = [int(m.group(1)) for m in re.finditer(r"\b(\d+)\.", content[7:])]
+ matches = [int(m.group(1)) for m in finditer(r"\b(\d+)\.", content[7:])]
logger.debug("standalone verse number and period matches: %s", matches)
is_ascending = all(
earlier < later for earlier, later in zip(matches, matches[1:])
@@ -454,53 +517,89 @@ def fix_standalone_verse_number_and_period(content: str) -> str:
logger.debug("is_ascending: %s", is_ascending)
num_matches = len(matches)
if (
- not re.compile(r"""\\v \d+""").search(
- context_for_standalone_verse_and_period
- )
+ not compile(r"""\\v \d+""").search(context_for_standalone_verse_and_period)
and not num_matches >= num_of_occurrences
) or is_ascending: # Check for non-ascending numbers
- return re.sub(
+ return sub(
pattern_matchers["fix_standalone_verse_number_and_period"],
r"\\v \1 \2",
content,
)
- else:
- logger.info(
- "Actually, we can't be certain it was a standalone verse number and period after all upon further checking"
- )
+ logger.info(
+ "Actually, we can't be certain it was a standalone verse number and period after all upon further checking"
+ )
return content
-def replace_n_with_v(content: str) -> str:
+def replace_n_with_v(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
"""Replace \n used mistakenly as verse markers with \v"""
- return re.sub(pattern_matchers["replace_n_with_v"], r"""\\v""", content)
+ return sub(pattern_matchers["replace_n_with_v"], r"\\v", content)
-def replace_cc_with_c(content: str) -> str:
+def replace_cc_with_c(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
"""
Replace two consecutive chapter markers with whitespace or newline between them with only one chapter marker
"""
- return re.sub(pattern_matchers["replace_cc_with_c"], r"\1" + "\n", content)
+ return sub(pattern_matchers["replace_cc_with_c"], r"\1" + "\n", content)
-def replace_vv_with_v(content: str) -> str:
+def replace_vv_with_v(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
"""Replace \v\v, caused by other correcting functions, with \v"""
- return re.sub(pattern_matchers["replace_vv_with_v"], "\\v", content)
+ return sub(pattern_matchers["replace_vv_with_v"], r"\\v", content)
-def replace_sv_with_s(content: str) -> str:
+def replace_sv_with_s(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
r"""Replace \s\v, caused by other correcting functions, with \s"""
- return re.sub(pattern_matchers["replace_sv_with_s"], "\\s", content)
+ return sub(pattern_matchers["replace_sv_with_s"], r"\\s", content)
-def fix_space_after_section_marker(content: str) -> str:
+def fix_space_after_section_marker(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
"""Reunite section marker with its value, caused by other correcting functions"""
- return re.sub(pattern_matchers["fix_space_after_section_marker"], r"\\s\1", content)
+ return sub(pattern_matchers["fix_space_after_section_marker"], r"\\s\1", content)
-def replace_qv_with_q(content: str) -> str:
+def replace_qv_with_q(
+ content: str,
+ pattern_matchers: Mapping[str, str] = PATTERN_MATCHERS,
+) -> str:
r"""Replace \q\v , caused by other correcting functions, with \q"""
- return re.sub(pattern_matchers["replace_qv_with_q"], r"\\q\1", content)
+ return sub(pattern_matchers["replace_qv_with_q"], r"\\q\1", content)
+
+
+RULES: list[Rule] = [
+ make_rule("fix_dot_after_verse_number", fix_dot_after_verse_number),
+ make_rule("fix_verse_marker_without_v", fix_verse_marker_without_v),
+ make_rule("fix_missing_space_before_number", fix_missing_space_before_number),
+ make_rule("fix_missing_space_after_number", fix_missing_space_after_number),
+ make_rule(
+ "fix_missing_space_before_verse_marker", fix_missing_space_before_verse_marker
+ ),
+ make_rule("fix_standalone_verse_numbers", fix_standalone_verse_numbers),
+ make_rule(
+ "fix_standalone_verse_number_and_period", fix_standalone_verse_number_and_period
+ ),
+ make_rule("replace_n_with_v", replace_n_with_v),
+ make_rule("replace_vv_with_v", replace_vv_with_v),
+ make_rule("replace_sv_with_s", replace_sv_with_s),
+ make_rule("fix_space_after_section_marker", fix_space_after_section_marker),
+ make_rule("replace_qv_with_q", replace_qv_with_q),
+ make_rule("replace_cc_with_c", replace_cc_with_c),
+]
def fix_usfm(
@@ -508,239 +607,22 @@ def fix_usfm(
lang_code: str,
resource_type: str,
book_code: str,
+ rules: Sequence[Rule] = RULES,
) -> str:
"""
Detect and correct many USFM structural issues in USFM source.
"""
- # logger.debug("Possibly defective USFM content: %s", usfm_content)
- corrected_usfm_content: str = usfm_content
- # NOTE This is called in a different place now, leaving commented out for now.
- # if compiled_patterns["remove_null_bytes_and_control_characters"].search(
- # corrected_usfm_content
- # ):
- # logger.debug(
- # "USFM defect, %s, detected for resource: %s-%s-%s, about to attempt fix...",
- # "remove_null_bytes_and_control_characters",
- # resource_lookup_dto.lang_code,
- # resource_lookup_dto.resource_type,
- # resource_lookup_dto.book_code,
- # )
- # corrected_usfm_content = remove_null_bytes_and_control_characters(
- # corrected_usfm_content
- # )
- if match := compiled_patterns["fix_dot_after_verse_number"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "fix_dot_after_verse_number",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_dot_after_verse_number(corrected_usfm_content)
- if match := compiled_patterns["fix_verse_marker_without_v"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "fix_verse_marker_without_v",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_verse_marker_without_v(corrected_usfm_content)
- if match := compiled_patterns["fix_missing_space_before_number"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "fix_missing_space_before_number",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_missing_space_before_number(corrected_usfm_content)
- if match := compiled_patterns["fix_missing_space_after_number"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "Potential USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, if confirmed, then will attempt fix...",
- "fix_missing_space_after_number",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_missing_space_after_number(corrected_usfm_content)
- if match := compiled_patterns["fix_missing_space_before_verse_marker"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "fix_missing_space_before_verse_marker",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_missing_space_before_verse_marker(
- corrected_usfm_content
- )
- if match := compiled_patterns["fix_standalone_verse_numbers"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "Possible USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, if confirmed, attempt to fix...",
- "fix_standalone_verse_numbers",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_standalone_verse_numbers(corrected_usfm_content)
- if match := compiled_patterns["fix_standalone_verse_number_and_period"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "Possible USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, if confirmed, attempt to fix...",
- "fix_standalone_verse_number_and_period",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_standalone_verse_number_and_period(
- corrected_usfm_content
- )
- if match := compiled_patterns["replace_n_with_v"].search(corrected_usfm_content):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "replace_n_with_v",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = replace_n_with_v(corrected_usfm_content)
- if match := compiled_patterns["replace_vv_with_v"].search(corrected_usfm_content):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "replace_vv_with_v",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = replace_vv_with_v(corrected_usfm_content)
- if match := compiled_patterns["replace_sv_with_s"].search(corrected_usfm_content):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "replace_sv_with_s",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = replace_sv_with_s(corrected_usfm_content)
- if match := compiled_patterns["fix_space_after_section_marker"].search(
- corrected_usfm_content
- ):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "fix_space_after_section_marker",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = fix_space_after_section_marker(corrected_usfm_content)
- if match := compiled_patterns["replace_qv_with_q"].search(corrected_usfm_content):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "replace_qv_with_q",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = replace_qv_with_q(corrected_usfm_content)
- if match := compiled_patterns["replace_cc_with_c"].search(corrected_usfm_content):
- logger.debug(
- "USFM defect %s detected, specifically %s, context: %s, for resource: %s-%s-%s, about to attempt fix...",
- "replace_cc_with_c",
- match.group(),
- corrected_usfm_content[
- max(0, match.start() - 5) : min(
- len(corrected_usfm_content), match.end() + 5
- )
- ],
- lang_code,
- resource_type,
- book_code,
- )
- corrected_usfm_content = replace_cc_with_c(corrected_usfm_content)
+ corrected_usfm_content = usfm_content
+ for rule_name, detect, fix, context_formatter in rules:
+ if match := detect(corrected_usfm_content):
+ log_detection(
+ rule_name,
+ match,
+ corrected_usfm_content,
+ lang_code,
+ resource_type,
+ book_code,
+ context_formatter,
+ )
+ corrected_usfm_content = fix(corrected_usfm_content)
return corrected_usfm_content
diff --git a/backend/doc/markdown_transforms/link_regexes.py b/backend/doc/markdown_transforms/link_regexes.py
index cf3c3d03..1258bf54 100644
--- a/backend/doc/markdown_transforms/link_regexes.py
+++ b/backend/doc/markdown_transforms/link_regexes.py
@@ -2,7 +2,6 @@
import re
-
# Handle TW wikilink inner text
TW_RC_LINK_RE = re.compile(
(
@@ -185,3 +184,9 @@
BC_MARKDOWN_LINK_RE = re.compile(
r"\[(?P.+?)\] *\(\.\.\/(?Particles.+?)\)"
)
+
+
+SEE_PARENTHETICAL_RE = re.compile(
+ r"\((?P[^\s():]+:)\s*.*?\)",
+ re.IGNORECASE,
+)
diff --git a/backend/doc/markdown_transforms/markdown_transformer.py b/backend/doc/markdown_transforms/markdown_transformer.py
index b0d1e66e..a3014a6d 100644
--- a/backend/doc/markdown_transforms/markdown_transformer.py
+++ b/backend/doc/markdown_transforms/markdown_transformer.py
@@ -8,6 +8,7 @@
from doc.domain.model import ResourceRequest
from doc.markdown_transforms.link_regexes import (
RC_QUESTION_LINK_RE,
+ SEE_PARENTHETICAL_RE,
TA_MARKDOWN_HTTPS_LINK_RE,
TA_PREFIXED_MARKDOWN_HTTPS_LINK_RE,
TA_PREFIXED_MARKDOWN_LINK_RE,
@@ -34,7 +35,6 @@
from doc.utils.file_utils import read_file
from doc.utils.tw_utils import localized_translation_word
-
logger = settings.logger(__name__)
TRANSLATION_WORD_ANCHOR_LINK_FMT_STR: str = "[{}](#{}-{})"
@@ -124,6 +124,8 @@ def transform_tw_links(
source = transform_rc_obe_tw_links(
source, lang_code, resource_requests, translation_words_dict
)
+
+ source = remove_see_colon_prefixed_parenthicals(source)
return source
@@ -158,6 +160,7 @@ def transform_ta_and_tn_links(
source = transform_tn_missing_book_code_markdown_links_no_paren(source)
source = transform_tn_obs_markdown_links(source)
source = transform_rc_question_links(source)
+ source = remove_see_colon_prefixed_parenthicals(source)
return source
@@ -1045,3 +1048,14 @@ def wiki_link_parser(
for link in finditer(wiki_link_re, source)
]
return links
+
+
+def remove_see_colon_prefixed_parenthicals(
+ source: str,
+ see_parenthetical_re: re.Pattern[str] = SEE_PARENTHETICAL_RE,
+) -> str:
+ """
+ Remove any parenthetical string beginning with localized '(See:'.
+ Example: '(Veja: foobar)' -> ''
+ """
+ return see_parenthetical_re.sub("", source)
diff --git a/backend/doc/utils/docx_util.py b/backend/doc/utils/docx_util.py
index 2f35821d..f51797a4 100644
--- a/backend/doc/utils/docx_util.py
+++ b/backend/doc/utils/docx_util.py
@@ -56,6 +56,33 @@ def _make_text_run(text: str) -> Element:
return r
+def override_and_clean_hyperlinks(doc: DocxDocument, style_id: str) -> None:
+ """
+ Finds all hyperlinks, strips any forced inline formatting (like blue color),
+ and applies the correct template character style ID using raw XML manipulation.
+ """
+ for paragraph in doc.paragraphs:
+ hyperlinks = paragraph._p.findall(
+ ".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hyperlink"
+ )
+ for hyperlink in hyperlinks:
+ runs = hyperlink.findall(qn("w:r"))
+ for r in runs:
+ rPr = r.find(qn("w:rPr"))
+ if rPr is None:
+ rPr = OxmlElement("w:rPr")
+ r.insert(0, rPr)
+ existing_style = rPr.find(qn("w:rStyle"))
+ if existing_style is not None:
+ rPr.remove(existing_style)
+ for child in list(rPr):
+ if child.tag in (qn("w:color"), qn("w:u"), qn("w:shd")):
+ rPr.remove(child)
+ rStyle = OxmlElement("w:rStyle")
+ rStyle.set(qn("w:val"), style_id)
+ rPr.insert(0, rStyle)
+
+
def _make_internal_hyperlink_element(text: str, bookmark_name: str) -> Element:
hyperlink = OxmlElement("w:hyperlink")
hyperlink.set(qn("w:anchor"), bookmark_name)
diff --git a/backend/doc/utils/tw_utils.py b/backend/doc/utils/tw_utils.py
index 23c36471..55692aea 100644
--- a/backend/doc/utils/tw_utils.py
+++ b/backend/doc/utils/tw_utils.py
@@ -23,7 +23,6 @@
)
from doc.utils.list_utils import unique_list_of_strings
-
logger = settings.logger(__name__)
TW = "tw"
@@ -181,8 +180,12 @@ def translation_words_content(
tw_book: TWBook,
content: str,
use_section_visual_separator: bool,
- tw_word_list_vertical: bool = settings.TW_WORD_LIST_VERTICAL,
+ link_rather_than_include_tw_definitions: bool = settings.LINK_RATHER_THAN_INCLUDE_TW_DEFINITIONS,
resource_type_name_fmt_str: str = settings.RESOURCE_TYPE_NAME_FMT_STR,
+ biel_tw_resource_path_fmt_str: str = settings.BIEL_TW_RESOURCE_URL_FMT_STR,
+ tw_resource_path_fmt_str: str = settings.TW_RESOURCE_URL_FMT_STR,
+ div_open: str = "",
+ div_close: str = "
",
) -> list[DocumentPart]:
is_rtl = tw_book and tw_book.lang_direction == LangDirEnum.RTL
document_parts: list[DocumentPart] = []
@@ -196,17 +199,19 @@ def translation_words_content(
use_section_visual_separator=False,
)
)
- if tw_word_list_vertical:
+ if link_rather_than_include_tw_definitions:
document_parts.append(
DocumentPart(
- content="\n"
- + "\n".join(
+ content=div_open
+ + ", ".join(
[
- f"- {localized_word}
"
+ biel_tw_resource_path_fmt_str.format(
+ tw_book.lang_code, localized_word
+ )
for localized_word, word in unique_words
]
)
- + "
",
+ + div_close,
is_rtl=is_rtl,
use_section_visual_separator=False,
)
@@ -214,14 +219,16 @@ def translation_words_content(
else:
document_parts.append(
DocumentPart(
- content=""
+ content=div_open
+ ", ".join(
[
- f"{localized_word}"
+ tw_resource_path_fmt_str.format(
+ tw_book.lang_code, word, localized_word
+ )
for localized_word, word in unique_words
]
)
- + "
",
+ + div_close,
is_rtl=is_rtl,
use_section_visual_separator=False,
)
diff --git a/backend/passages/domain/document_generator.py b/backend/passages/domain/document_generator.py
index 4a79e56d..841d7334 100644
--- a/backend/passages/domain/document_generator.py
+++ b/backend/passages/domain/document_generator.py
@@ -8,7 +8,9 @@
from doc.domain.bible_books import BOOK_NAMES
from doc.domain.email_utils import send_email_with_attachment, should_send_email
from doc.domain.model import Attachment, USFMBook
-from doc.domain.parsing import split_chapter_into_verses, usfm_book_content
+from doc.domain.parsing import (
+ usfm_book_content,
+)
from doc.domain.resource_lookup import (
book_codes_for_lang_from_usfm_only,
prepare_resource_filepath,
@@ -29,7 +31,7 @@
Passage,
BibleReferenceWithAvailability,
)
-from passages.domain.parser import verse_text_html
+from passages.domain.parser import split_chapter_into_verses, verse_text_html
from passages.domain.stet_verse_list_parser import BOOK_INDEX, parse_bible_blocks
from passages.utils.docx_utils import add_footer, add_header
from pydantic import Json
@@ -101,6 +103,9 @@ def get_usfm_books_and_usfm_resource_type(
bible_references_with_availability: list[BibleReferenceWithAvailability],
lang_code: str,
usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
+ languages_where_non_ulb_preferred: Sequence[
+ str
+ ] = settings.LANGUAGES_WHERE_NON_ULB_PREFERRED,
) -> tuple[list[USFMBook], str]:
# Invariant: book codes are only those that were available from USFM resources
book_codes = list(
@@ -128,10 +133,16 @@ def get_usfm_books_and_usfm_resource_type(
)
usfm_books = []
usfm_resource_type = ""
- if ulb_usfm_resource_types: # Prefer ulb if available
- usfm_resource_type = ulb_usfm_resource_types[0]
- elif usfm_resource_types:
- usfm_resource_type = usfm_resource_types[0]
+ if lang_code not in languages_where_non_ulb_preferred:
+ if ulb_usfm_resource_types: # Prefer ulb if available
+ usfm_resource_type = ulb_usfm_resource_types[0]
+ elif usfm_resource_types:
+ usfm_resource_type = usfm_resource_types[0]
+ else:
+ if usfm_resource_types: # Prefer non-ulb if available
+ usfm_resource_type = usfm_resource_types[0]
+ elif ulb_usfm_resource_types:
+ usfm_resource_type = ulb_usfm_resource_types[0]
if usfm_resource_type:
usfm_book = None
for book_code in book_codes:
@@ -197,22 +208,22 @@ def generate_docx_document(
)
)
current_task.update_state(state="Assembling content")
- passages_lang0 = get_passages(
+ lang0_passages = get_passages(
bible_references_with_availability_lang0,
usfm_resource_type_lang0,
usfm_books_lang0,
)
- passages_lang1 = []
+ lang1_passages = []
if lang1_code:
- passages_lang1 = get_passages(
+ lang1_passages = get_passages(
bible_references_with_availability_lang1,
usfm_resource_type_lang1,
usfm_books_lang1,
)
current_task.update_state(state="Converting to Docx")
generate_docx(
- passages_lang0,
- passages_lang1,
+ lang0_passages,
+ lang1_passages,
docx_filepath_,
lang0_code,
lang0_name,
@@ -223,8 +234,8 @@ def generate_docx_document(
def generate_docx(
- passages_lang0: list[Passage],
- passages_lang1: list[Passage],
+ lang0_passages: list[Passage],
+ lang1_passages: list[Passage],
docx_filepath: str,
lang0_code: str,
lang0_name: str,
@@ -245,9 +256,9 @@ def generate_docx(
html_to_docx = HtmlToDocx()
has_lang1 = lang1_code is not None and lang1_name is not None
if has_lang1:
- assert len(passages_lang0) == len(passages_lang1), (
+ assert len(lang0_passages) == len(lang1_passages), (
f"Passage count mismatch: "
- f"{len(passages_lang0)} vs {len(passages_lang1)}"
+ f"{len(lang0_passages)} vs {len(lang1_passages)}"
)
columns: list[str] = ["lang0"]
if has_lang1:
@@ -271,9 +282,9 @@ def generate_docx(
table.columns[i].width = w
col_index = {name: i for i, name in enumerate(columns)}
pairs = (
- zip(passages_lang0, passages_lang1)
+ zip(lang0_passages, lang1_passages)
if has_lang1
- else ((p, None) for p in passages_lang0)
+ else ((p, None) for p in lang0_passages)
)
for p0, p1 in pairs:
row = table.add_row()
diff --git a/backend/passages/domain/parser.py b/backend/passages/domain/parser.py
index 455f3366..210704d4 100644
--- a/backend/passages/domain/parser.py
+++ b/backend/passages/domain/parser.py
@@ -1,12 +1,13 @@
+from re import split, sub
from typing import Mapping
+from bs4 import BeautifulSoup, NavigableString
from doc.config import settings
from doc.domain.bible_books import BOOK_CHAPTER_VERSES
-from doc.domain.model import USFMBook
+from doc.domain.model import USFMBook, USFMChapter
from doc.domain.parsing import lookup_verse_text
from doc.reviewers_guide.model import BibleReference
-
logger = settings.logger(__name__)
@@ -102,3 +103,63 @@ def verse_text_html(
f'{bible_reference.start_chapter_verse_ref.strip()}{verse_text___}'
)
return "".join(verse_text)
+
+
+def split_chapter_into_verses(chapter: USFMChapter) -> dict[str, str]:
+ # Sample HTML content with multiple verse elements
+ # html_content = '''
+ #
+ # 19
+ # For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
+ #
+ #
+ #
+ #
+ # 20
+ # I have been crucified with Christ and I no longer live, but Christ lives in me. The life I now live in the body, I live by faith in the Son of God, who loved me and gave himself for me.
+ #
+ #
+ #
+ # '''
+ verse_dict: dict[str, str] = {}
+ soup = BeautifulSoup(chapter.content, "html.parser")
+ for verse_span in soup.find_all("span", class_="verse"):
+ versemarker = verse_span.find("sup", class_="versemarker")
+ if not versemarker or not versemarker.get_text(strip=True):
+ continue
+ verse_number = versemarker.get_text(strip=True)
+ # Remove verse marker
+ versemarker.decompose()
+ # Remove footnote callers
+ for caller in verse_span.find_all("sup", class_="caller"):
+ caller.decompose()
+ # Fix spacing issue for poetry divs
+ for poetry_div in verse_span.find_all(
+ "div", class_=lambda c: c and c.startswith("poetry-")
+ ):
+ poetry_div.insert_before(NavigableString(" "))
+ # Handle fr f10 word-entry tags
+ for we in verse_span.find_all("span", class_="word-entry"):
+ we.unwrap()
+ # Get inner HTML of the verse span
+ verse_text = "".join(str(child) for child in verse_span.contents).strip()
+ # verse_text = clean_verse_html(verse_text)
+ verse_dict[verse_number] = verse_text
+ return verse_dict
+
+
+def clean_verse_html(
+ raw_verse: str,
+ empty_paragraph: str = "",
+ sectionhead5_element: str = '',
+) -> str:
+ cleaned_html = raw_verse
+ cleaned_html = sub(r"\s+([,;:.!?])", r"\1", cleaned_html)
+ cleaned_html = sub(r"\s+'", "'", cleaned_html)
+ cleaned_html = sub(r"'\s+", "'", cleaned_html)
+ cleaned_html = sub(r"\s*-\s*", "-", cleaned_html)
+ cleaned_html = sub(r"\s{2,}", " ", cleaned_html).strip()
+ cleaned_html = cleaned_html.replace(empty_paragraph, "").replace(
+ sectionhead5_element, ""
+ )
+ return cleaned_html
diff --git a/backend/stet/data/stet_en.docx b/backend/stet/data/stet_en.docx
index 22b097f7..73805560 100644
Binary files a/backend/stet/data/stet_en.docx and b/backend/stet/data/stet_en.docx differ
diff --git a/backend/stet/data/stet_es-419.docx b/backend/stet/data/stet_es-419.docx
index a1faade4..464e4be7 100644
Binary files a/backend/stet/data/stet_es-419.docx and b/backend/stet/data/stet_es-419.docx differ
diff --git a/backend/stet/data/stet_fr.docx b/backend/stet/data/stet_fr.docx
index 536cacc3..e1ab0a9b 100644
Binary files a/backend/stet/data/stet_fr.docx and b/backend/stet/data/stet_fr.docx differ
diff --git a/backend/stet/data/stet_gu.docx b/backend/stet/data/stet_gu.docx
new file mode 100644
index 00000000..7e3601cd
Binary files /dev/null and b/backend/stet/data/stet_gu.docx differ
diff --git a/backend/stet/data/stet_rmn-x-yerliroman.docx b/backend/stet/data/stet_rmn-x-yerliroman.docx
new file mode 100644
index 00000000..31bd5490
Binary files /dev/null and b/backend/stet/data/stet_rmn-x-yerliroman.docx differ
diff --git a/backend/stet/data/stet_ru.docx b/backend/stet/data/stet_ru.docx
new file mode 100644
index 00000000..1ca59be8
Binary files /dev/null and b/backend/stet/data/stet_ru.docx differ
diff --git a/backend/stet/data/stet_sw.docx b/backend/stet/data/stet_sw.docx
index 3f20f1d0..1f18e2ac 100644
Binary files a/backend/stet/data/stet_sw.docx and b/backend/stet/data/stet_sw.docx differ
diff --git a/backend/stet/data/stet_tpi.docx b/backend/stet/data/stet_tpi.docx
index b149c3b6..c0f7df2d 100644
Binary files a/backend/stet/data/stet_tpi.docx and b/backend/stet/data/stet_tpi.docx differ
diff --git a/backend/stet/data/stet_vi.docx b/backend/stet/data/stet_vi.docx
new file mode 100644
index 00000000..aa0a89e2
Binary files /dev/null and b/backend/stet/data/stet_vi.docx differ
diff --git a/backend/stet/domain/document_generator.py b/backend/stet/domain/document_generator.py
index b45feb79..128d2ab7 100644
--- a/backend/stet/domain/document_generator.py
+++ b/backend/stet/domain/document_generator.py
@@ -10,7 +10,6 @@
from doc.domain.model import Attachment
from doc.domain.parsing import (
lookup_verse_text,
- split_chapter_into_verses,
usfm_book_content,
)
from doc.domain.resource_lookup import (
@@ -22,13 +21,15 @@
from doc.utils.file_utils import docx_filepath, file_needs_update
from doc.utils.text_utils import maybe_correct_book_name
from docx import Document
+from docx.enum.table import WD_TABLE_ALIGNMENT
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
+from docx.shared import Length, Mm, Pt
from html4docx import HtmlToDocx # type: ignore
from pydantic import Json
from stet.domain.model import VerseEntry, WordEntry
-from stet.domain.parser import get_word_entry_dtos
+from stet.domain.parser import get_word_entry_dtos, split_chapter_into_verses
from stet.domain.strings import (
LOCALIZED_DATE_FORMAT_STRINGS,
TRANSLATED_FOOTER_PHRASES_TABLE,
@@ -61,6 +62,9 @@ def generate_docx_document(
resource_type_codes_and_names: Mapping[
str, str
] = settings.RESOURCE_TYPE_CODES_AND_NAMES,
+ languages_where_non_ulb_preferred: Sequence[
+ str
+ ] = settings.LANGUAGES_WHERE_NON_ULB_PREFERRED,
) -> str:
"""
Generate the scriptural terms evaluation document.
@@ -116,14 +120,26 @@ def generate_docx_document(
target_usfm_books = []
lang0_usfm_resource_type = ""
lang1_usfm_resource_type = ""
- if lang0_ulb_usfm_resource_types: # Prefer ulb if available
- lang0_usfm_resource_type = lang0_ulb_usfm_resource_types[0]
- elif lang0_usfm_resource_types:
- lang0_usfm_resource_type = lang0_usfm_resource_types[0]
- if lang1_ulb_usfm_resource_types: # Prefer ulb if available
- lang1_usfm_resource_type = lang1_ulb_usfm_resource_types[0]
- elif lang1_usfm_resource_types:
- lang1_usfm_resource_type = lang1_usfm_resource_types[0]
+ if lang0_code not in languages_where_non_ulb_preferred:
+ if lang0_ulb_usfm_resource_types: # Prefer ulb if available
+ lang0_usfm_resource_type = lang0_ulb_usfm_resource_types[0]
+ elif lang0_usfm_resource_types:
+ lang0_usfm_resource_type = lang0_usfm_resource_types[0]
+ else:
+ if lang0_usfm_resource_types: # Prefer non-ulb if available
+ lang0_usfm_resource_type = lang0_usfm_resource_types[0]
+ elif lang0_ulb_usfm_resource_types:
+ lang0_usfm_resource_type = lang0_ulb_usfm_resource_types[0]
+ if lang1_code not in languages_where_non_ulb_preferred:
+ if lang1_ulb_usfm_resource_types: # Prefer ulb if available
+ lang1_usfm_resource_type = lang1_ulb_usfm_resource_types[0]
+ elif lang1_usfm_resource_types:
+ lang1_usfm_resource_type = lang1_usfm_resource_types[0]
+ else:
+ if lang1_usfm_resource_types: # Prefer non-ulb if available
+ lang1_usfm_resource_type = lang1_usfm_resource_types[0]
+ elif lang1_ulb_usfm_resource_types:
+ lang1_usfm_resource_type = lang1_ulb_usfm_resource_types[0]
if lang0_usfm_resource_type and lang1_usfm_resource_type:
source_usfm_book = None
target_usfm_book = None
@@ -146,10 +162,11 @@ def generate_docx_document(
lang0_resource_dir,
False,
)
- for chapter_num_, chapter_ in source_usfm_book.chapters.items():
- source_usfm_book.chapters[chapter_num_].verses = (
- split_chapter_into_verses(chapter_)
- )
+ for (
+ chapter_num_,
+ chapter_,
+ ) in source_usfm_book.chapters.items():
+ chapter_.verses = split_chapter_into_verses(chapter_)
source_usfm_books.append(source_usfm_book)
lang1_resource_lookup_dto_ = resource_lookup_dto(
lang1_code, lang1_usfm_resource_type, book_code
@@ -166,10 +183,11 @@ def generate_docx_document(
lang1_resource_dir,
False,
)
- for chapter_num_, chapter_ in target_usfm_book.chapters.items():
- target_usfm_book.chapters[chapter_num_].verses = (
- split_chapter_into_verses(chapter_)
- )
+ for (
+ chapter_num_,
+ chapter_,
+ ) in target_usfm_book.chapters.items():
+ chapter_.verses = split_chapter_into_verses(chapter_)
target_usfm_books.append(target_usfm_book)
# Count total occurrences per reference (using source_reference as key)
reference_counter: Counter[str] = Counter()
@@ -296,56 +314,83 @@ def generate_docx(
translated_footer_phrases_table: dict[str, str] = TRANSLATED_FOOTER_PHRASES_TABLE,
localized_date_format_strings: dict[str, str] = LOCALIZED_DATE_FORMAT_STRINGS,
translated_header_phrases_table: dict[str, str] = TRANSLATED_HEADER_PHRASES_TABLE,
+ margin_width: Length = Pt(54),
+ a4_width: Length = Mm(210),
+ a4_height: Length = Mm(297),
) -> None:
"""
- Generates a DOCX document from a list of word entries and saves it to the given file path.
- :param word_entries: A list of word entries containing the word, strongs numbers, definition, and verses.
- :param docx_filepath: The file path where the generated DOCX document will be saved.
- :param lang0_code: Source language code for the document header.
- :param lang1_code: Target language code for the document header.
+ Generates a DOCX document optimized for A4 paper printing from a list of word entries.
"""
doc = Document()
+ section = doc.sections[0]
+ section.page_width = a4_width
+ section.page_height = a4_height
+ section.left_margin = margin_width
+ section.right_margin = margin_width
+ section.top_margin = Pt(54)
+ section.bottom_margin = Pt(54)
+ printable_width_emu: int = int(a4_width) - (2 * int(margin_width))
+ printable_width: Length = Length(printable_width_emu)
+ col_widths: list[Length] = [
+ Length(int(printable_width_emu * 0.45)),
+ Length(int(printable_width_emu * 0.45)),
+ Length(int(printable_width_emu * 0.10)),
+ ]
html_to_docx = HtmlToDocx()
for word_entry in word_entries:
- # Add the word heading
heading: str = (
f"{','.join(word_entry.words)} ({word_entry.strongs_numbers})"
if word_entry.strongs_numbers
else "".join(word_entry.words)
)
doc.add_heading(heading, level=1)
- # Convert the HTML definition to DOCX content
if word_entry.definition:
html_to_docx.add_html_to_document(word_entry.definition, doc)
- # Create a table with three columns
table = doc.add_table(rows=1, cols=3)
table.style = "Table Grid"
- # Set the header of the table and apply bold formatting
- hdr_cells = table.rows[0].cells
+ table.alignment = WD_TABLE_ALIGNMENT.CENTER
+ table.autofit = False
+ table.allow_autofit = False
+ table.width = printable_width
+ for i, w in enumerate(col_widths):
+ table.columns[i].width = w
+ hdr_row = table.rows[0]
+ trPr = hdr_row._tr.get_or_add_trPr()
+ trPr.append(OxmlElement("w:tblHeader"))
+ hdr_cells = hdr_row.cells
+ for i, cell in enumerate(hdr_cells):
+ cell.width = col_widths[i]
hdr_cells[0].text = translated_table_column_headers[lang0_code][0]
hdr_cells[1].text = translated_table_column_headers[lang0_code][1]
hdr_cells[2].text = translated_table_column_headers[lang0_code][2]
hdr_cells[2].paragraphs[0].alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
for hdr_cell in hdr_cells:
hdr_cell.paragraphs[0].runs[0].bold = True
- # Add verses to the table
for verse in word_entry.verses:
- # Row for references
- row_cells = table.add_row().cells
- source_ref_display = verse.source_reference
+ ref_row = table.add_row()
+ trPr = ref_row._tr.get_or_add_trPr()
+ trPr.append(OxmlElement("w:cantSplit"))
+ row_cells = ref_row.cells
+ for i, w in enumerate(col_widths):
+ row_cells[i].width = w
+ source_paragraph = row_cells[0].paragraphs[0]
+ source_run = source_paragraph.add_run(verse.source_reference)
+ source_run.bold = True
if verse.occurrence_total > 1:
- source_ref_display += (
+ occurrence_run = source_paragraph.add_run(
f" ({verse.occurrence_index}/{verse.occurrence_total})"
)
- target_ref_display = verse.target_reference
+ occurrence_run.bold = True
+ occurrence_run.italic = True
+ target_paragraph = row_cells[1].paragraphs[0]
+ target_run = target_paragraph.add_run(verse.target_reference)
+ target_run.bold = True
if verse.occurrence_total > 1:
- target_ref_display += (
+ occurrence_run = target_paragraph.add_run(
f" ({verse.occurrence_index}/{verse.occurrence_total})"
)
- source_run = row_cells[0].paragraphs[0].add_run(source_ref_display)
- source_run.bold = True
- target_run = row_cells[1].paragraphs[0].add_run(target_ref_display)
- target_run.bold = True
+ occurrence_run.bold = True
+ occurrence_run.italic = True
status_run = (
row_cells[2]
.paragraphs[0]
@@ -353,37 +398,36 @@ def generate_docx(
)
status_run.bold = True
row_cells[2].paragraphs[0].alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
- # Row for texts
- row_cells = table.add_row().cells
- # Process HTML content in source_text and highlight keyword
+ text_row = table.add_row()
+ trPr = text_row._tr.get_or_add_trPr()
+ trPr.append(OxmlElement("w:cantSplit"))
+ row_cells = text_row.cells
+ for i, w in enumerate(col_widths):
+ row_cells[i].width = w
source_paragraph = row_cells[0].paragraphs[0]
- source_paragraph.paragraph_format.line_spacing = 2.0 # Adjust line spacing
+ source_paragraph.paragraph_format.line_spacing = 1.3
if verse.source_has_preformatted_bolding:
add_preformatted_html_to_docx(verse.source_text, source_paragraph)
elif len(word_entry.bolded_phrases) > 0:
add_highlighted_html_to_docx_for_words(
verse.source_text, source_paragraph, word_entry.bolded_phrases
)
- else: # Bolded phrases in 4th column were not provided
+ else:
add_highlighted_html_to_docx_for_words(
verse.source_text, source_paragraph, word_entry.words
)
- # Add target_text with wider line spacing
target_paragraph = row_cells[1].paragraphs[0]
- target_paragraph.paragraph_format.line_spacing = 2.0 # Adjust line spacing
+ target_paragraph.paragraph_format.line_spacing = 1.3
add_plain_html_to_docx(verse.target_text, target_paragraph)
- # Vertically centered Unicode checkbox
checkbox_cell = row_cells[2]
checkbox_paragraph = checkbox_cell.paragraphs[0]
checkbox_paragraph.text = "\u2610"
checkbox_paragraph.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
- tc = checkbox_cell._tc # Access the XML element of the table cell
- tcPr = tc.get_or_add_tcPr() # Get or add the cell properties
- vAlign = OxmlElement("w:vAlign") # Create the vertical alignment element
- vAlign.set(qn("w:val"), "center") # Set alignment to "center"
- tcPr.append(vAlign) # Append the vertical alignment to cell properties
- # Adjust column widths to prioritize the first two columns
- adjust_table_columns(table)
+ tc = checkbox_cell._tc
+ tcPr = tc.get_or_add_tcPr()
+ vAlign = OxmlElement("w:vAlign")
+ vAlign.set(qn("w:val"), "center")
+ tcPr.append(vAlign)
footer_phrase = translated_footer_phrases_table[lang0_code]
current_datetime = datetime.now().strftime(
localized_date_format_strings[lang0_code]
diff --git a/backend/stet/domain/parser.py b/backend/stet/domain/parser.py
index 849aba75..e6ac3751 100644
--- a/backend/stet/domain/parser.py
+++ b/backend/stet/domain/parser.py
@@ -1,17 +1,24 @@
-import re
+from re import (
+ DOTALL,
+ compile,
+ match,
+ split,
+ sub,
+)
+from bs4 import BeautifulSoup, NavigableString
from doc.config import settings
from doc.domain.bible_books import BOOK_NAMES
+from doc.domain.model import USFMChapter
from doc.domain.resource_lookup import book_codes_for_lang_from_usfm_only
from docx import Document
from stet.domain.model import VerseReferenceDto, WordEntryDto
from stet.utils.util import is_valid_int
-
logger = settings.logger(__name__)
-_RV_BLOCK_PATTERN = re.compile(r"(.*?)\s*(.*?)", re.DOTALL)
-_REF_PATTERN = re.compile(r"^(.*) (\d+):([0-9,\- ]+)\s?(\(.*\))?$")
+_RV_BLOCK_PATTERN = compile(r"(.*?)\s*(.*?)", DOTALL)
+_REF_PATTERN = compile(r"^(.*) (\d+):([0-9,\- ]+)\s?(\(.*\))?$")
def _parse_ref_to_dto(
@@ -24,11 +31,11 @@ def _parse_ref_to_dto(
lang0_book_codes_and_names__: list[tuple[str, str]],
source_text_with_bolding: str | None = None,
) -> VerseReferenceDto | None:
- match = _REF_PATTERN.match(reference_)
- if not match:
+ match_ = _REF_PATTERN.match(reference_)
+ if not match_:
logger.warning("Couldn't parse %s", reference_)
return None
- book_name = match.group(1).replace("\n", "")
+ book_name = match_.group(1).replace("\n", "")
book_codes_and_names_ = [
(bc, bn) for bc, bn in lang0_book_codes_and_names if bn == book_name
]
@@ -39,11 +46,12 @@ def _parse_ref_to_dto(
book_code_and_name_ = book_codes_and_names_[0] if book_codes_and_names_ else None
if book_code_and_name_:
lang0_book_codes_and_names__.append(book_code_and_name_)
- chapter_num = int(match.group(2))
- verses = match.group(3)
- comment = match.group(4)
+ chapter_num = int(match_.group(2))
+ verses = match_.group(3)
+ comment = match_.group(4)
source_reference = (
- f"{book_name} {chapter_num}:{verses}{comment}" if comment
+ f"{book_name} {chapter_num}:{verses}{comment}"
+ if comment
else f"{book_name} {chapter_num}:{verses}"
)
lang0_book_code = book_code_and_name_[0] if book_code_and_name_ else ""
@@ -59,7 +67,7 @@ def _parse_ref_to_dto(
if is_valid_int(verse_ref):
valid_verse_refs.append(str(verse_ref))
continue
- vm = re.match(r"(\d+)-(\d+)", verse_ref)
+ vm = match(r"(\d+)-(\d+)", verse_ref)
if vm:
for verse_num in range(int(vm.group(1)), int(vm.group(2)) + 1):
valid_verse_refs.append(str(verse_num))
@@ -149,12 +157,12 @@ def get_word_entry_dtos(
# Create entry item
word_entry_dto = WordEntryDto()
# Extract word from 1st column
- match = re.match(r"(.*)(\n)?(.*)?", row.cells[0].text)
- if not match:
+ match_ = match(r"(.*)(\n)?(.*)?", row.cells[0].text)
+ if not match_:
raise ValueError(f"Couldn't parse word(s): {row.cells[0].text}")
- words = match.group(1)
+ words = match_.group(1)
word_entry_dto.words = [word.strip() for word in words.split(",")]
- raw_strongs = match.group(3)
+ raw_strongs = match_.group(3)
word_entry_dto.strongs_numbers = raw_strongs.strip()
definition = ""
previous_paragraph_style_name = ""
@@ -203,3 +211,63 @@ def get_word_entry_dtos(
]
word_entry_dtos.append(word_entry_dto)
return word_entry_dtos, list(set(lang0_book_codes_and_names__))
+
+
+def split_chapter_into_verses(chapter: USFMChapter) -> dict[str, str]:
+ # Sample HTML content with multiple verse elements
+ # html_content = '''
+ #
+ # 19
+ # For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
+ #
+ #
+ #
+ #
+ # 20
+ # I have been crucified with Christ and I no longer live, but Christ lives in me. The life I now live in the body, I live by faith in the Son of God, who loved me and gave himself for me.
+ #
+ #
+ #
+ # '''
+ verse_dict: dict[str, str] = {}
+ soup = BeautifulSoup(chapter.content, "html.parser")
+ for verse_span in soup.find_all("span", class_="verse"):
+ versemarker = verse_span.find("sup", class_="versemarker")
+ if not versemarker or not versemarker.get_text(strip=True):
+ continue
+ verse_number = versemarker.get_text(strip=True)
+ # Remove verse marker
+ versemarker.decompose()
+ # Remove footnote callers
+ for caller in verse_span.find_all("sup", class_="caller"):
+ caller.decompose()
+ # Fix spacing issue for poetry divs
+ for poetry_div in verse_span.find_all(
+ "div", class_=lambda c: c and c.startswith("poetry-")
+ ):
+ poetry_div.insert_before(NavigableString(" "))
+ # Handle fr f10 word-entry tags
+ for we in verse_span.find_all("span", class_="word-entry"):
+ we.unwrap()
+ # Get inner HTML of the verse span
+ verse_text = "".join(str(child) for child in verse_span.contents).strip()
+ verse_text = clean_verse_html(verse_text)
+ verse_dict[verse_number] = verse_text
+ return verse_dict
+
+
+def clean_verse_html(
+ raw_verse: str,
+ empty_paragraph: str = "",
+ sectionhead5_element: str = '',
+) -> str:
+ cleaned_html = raw_verse
+ cleaned_html = sub(r"\s+([,;:.!?])", r"\1", cleaned_html)
+ cleaned_html = sub(r"\s+'", "'", cleaned_html)
+ cleaned_html = sub(r"'\s+", "'", cleaned_html)
+ cleaned_html = sub(r"\s*-\s*", "-", cleaned_html)
+ cleaned_html = sub(r"\s{2,}", " ", cleaned_html).strip()
+ cleaned_html = cleaned_html.replace(empty_paragraph, "").replace(
+ sectionhead5_element, ""
+ )
+ return cleaned_html
diff --git a/backend/stet/domain/resource_lookup.py b/backend/stet/domain/resource_lookup.py
new file mode 100644
index 00000000..d8db861e
--- /dev/null
+++ b/backend/stet/domain/resource_lookup.py
@@ -0,0 +1,55 @@
+from typing import Sequence
+
+from doc.config import settings
+from doc.domain.resource_lookup import fetch_source_data
+from doc.utils.list_utils import unique_tuples
+
+# List of languages which do not have USFM available for NT books. We use this
+# to filter these out of STET's list of source and target
+# languages so that the user doesn't have the frustrating experience of
+# selecting a language which might have OT USFM resources available but
+# not NT USFM so that when their resulting doc is generated no scripture is
+# present. It makes it seem like a bug in STET and is bad UX.
+LANG_CODES_WITH_NO_NT_USFM: frozenset[str] = frozenset(["ru"])
+
+logger = settings.logger(__name__)
+
+
+def lang_codes_and_names_having_usfm(
+ lang_code_filter_list: frozenset[str] = LANG_CODES_WITH_NO_NT_USFM,
+ gateway_languages: frozenset[str] = settings.GATEWAY_LANGUAGES,
+) -> Sequence[tuple[str, str, bool]]:
+ """
+ >>> from doc.domain import resource_lookup
+ >>> ();result = resource_lookup.lang_codes_and_names_having_usfm();() # doctest: +ELLIPSIS
+ (...)
+ >>> result[0]
+ ('abz', 'Abui', False)
+ >>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm() if not lang_code_and_name[2]]
+ >>> sorted(heart_lang_codes)[0]
+ 'aao'
+ """
+ data = fetch_source_data()
+ values = []
+ if data is None or not data.git_repo:
+ logger.info("Data API is down or no git_repo found!")
+ return []
+ try:
+ for repo_info in data.git_repo:
+ language_info = repo_info.content
+ language = language_info.language
+ ietf_code = language.ietf_code
+ english_name = language.english_name if language.english_name else ""
+ localized_name = language.national_name
+ is_gateway = ietf_code in gateway_languages
+ if ietf_code not in lang_code_filter_list:
+ if english_name in localized_name:
+ values.append((ietf_code, localized_name, is_gateway))
+ else:
+ values.append(
+ (ietf_code, f"{localized_name} ({english_name})", is_gateway)
+ )
+ except Exception:
+ logger.exception("Failed due to the following exception.")
+ unique_values = unique_tuples(values)
+ return sorted(unique_values, key=lambda value: value[1])
diff --git a/backend/stet/domain/strings.py b/backend/stet/domain/strings.py
index 28ddbe95..ec2902fc 100644
--- a/backend/stet/domain/strings.py
+++ b/backend/stet/domain/strings.py
@@ -2,34 +2,60 @@
"en": "Spiritual Terms Evaluation Tool (STET)",
"es-419": "Herramienta de Evaluación de Términos Espirituales (STET)",
"fr": "Évaluation des Termes Spirituels (STET)",
+ "gu": "આધ્યાત્મિક શબ્દો મૂલ્યાંકન સાધન (STET)",
"pt-br": "Ferramenta de Avaliação de Termos Espirituais (STET)",
+ "rmn-x-yerliroman": "Инструмент за оценка на духовни термини (STET)",
+ "ru": "Инструмент оценки духовных терминов (STET)",
"sw": "Chombo cha Kutathmini Maneno ya Kiroho (STET)",
"tpi": "Tul bilong skelim ol spirit tok bilong buk trenslesen (STET)",
+ "vi": "Công cụ Đánh giá Thuật ngữ Tâm linh (STET)",
}
TRANSLATED_FOOTER_PHRASES_TABLE: dict[str, str] = {
"en": "Generated on",
"es-419": "Generado el",
"fr": "Généré le",
+ "gu": "તૈયાર કરેલ તારીખ",
"pt-br": "Gerado em",
+ "rmn-x-yerliroman": "Генерирано на",
+ "ru": "Сформировано",
"sw": "Imetolewa tarehe",
"tpi": "Wok i bin kamap long",
+ "vi": "Được tạo vào",
}
LOCALIZED_DATE_FORMAT_STRINGS: dict[str, str] = {
"en": "%m/%d/%Y %H:%M:%S",
"es-419": "%d/%m/%Y %H:%M:%S",
"fr": "%d/%m/%Y %H:%M:%S",
+ "gu": "%d/%m/%Y %H:%M:%S",
"pt-br": "%d/%m/%Y %H:%M:%S",
+ "rmn-x-yerliroman": "%d.%m.%Y %H:%M:%S",
+ "ru": "%d.%m.%Y %H:%M:%S",
"sw": "%d/%m/%Y %H:%M:%S",
"tpi": "%d/%m/%Y %H:%M:%S",
+ "vi": "%d/%m/%Y %H:%M:%S",
}
TRANSLATED_TABLE_COLUMN_HEADERS = {
"en": ("Source Reference", "Target Reference", "Status", "OK"),
"es-419": ("Fuente", "Idioma Materna", "Estado", "OK"),
"fr": ("Référence de source", "Référence Cible", "Statut", "OK"),
+ "gu": ("સ્ત્રોત લખાણનો સંદર્ભ", "લક્ષ્ય લખાણનો સંદર્ભ", "સ્થિતિ", "બરાબર છે"),
"pt-br": ("Referência de Origem", "Referência de Destino", "Status", "OK"),
+ "rmn-x-yerliroman": (
+ "Препратка към изходния текст – текста на български",
+ "Препратка към преведения текст – текста на цигански",
+ "състояние",
+ "OK",
+ ),
+ "ru": (
+ "Ссылка на исходный текст",
+ "Ссылка на переведенный текст",
+ "статус",
+ "Удовл",
+ ),
"sw": ("Marejeo Chanzo", "Marejeo Lengwa", "Hali", "OK"),
"tpi": ("Narapela baibel ves", "Tokples ves", "Sek", "OK"),
+ "vi": ("Tham chiếu nguồn", "Tham chiếu đích", "Trạng thái", "Đạt"),
}
diff --git a/backend/stet/entrypoints/routes.py b/backend/stet/entrypoints/routes.py
index ce0501d9..5a2f5e97 100644
--- a/backend/stet/entrypoints/routes.py
+++ b/backend/stet/entrypoints/routes.py
@@ -4,7 +4,7 @@
import celery.states
from celery.result import AsyncResult
from doc.config import settings
-from doc.domain import resource_lookup
+from stet.domain import resource_lookup
from docx import Document
from fastapi import APIRouter, Request, HTTPException, status
from fastapi.responses import JSONResponse
diff --git a/backend/stet/utils/docx_utils.py b/backend/stet/utils/docx_utils.py
index 0bf56588..d10c0c2d 100644
--- a/backend/stet/utils/docx_utils.py
+++ b/backend/stet/utils/docx_utils.py
@@ -1,23 +1,25 @@
from __future__ import annotations
import re
-from typing import Optional, cast, TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional, cast
from docx import Document
from docx.document import Document as DocxDocument
-from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_PARAGRAPH_ALIGNMENT
from docx.enum.section import WD_SECTION
+from docx.enum.text import (
+ WD_ALIGN_PARAGRAPH,
+ WD_PARAGRAPH_ALIGNMENT,
+ WD_TAB_ALIGNMENT,
+ WD_TAB_LEADER,
+)
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from docx.oxml.table import CT_Tc
-from docx.shared import Pt, RGBColor
-from docx.table import Table
+from docx.shared import Mm, Pt, RGBColor
+from docx.table import Table, _Cell, _Row
from docx.text.paragraph import Paragraph
from html4docx import HtmlToDocx # type: ignore[import-untyped]
-
-from docx.table import _Cell, _Row
-
if TYPE_CHECKING:
from typing import TypeAlias
@@ -193,23 +195,29 @@ def add_lined_page_at_end(doc: DocxDocument) -> DocxDocument:
:param doc: The Word document to which the ruled page will be added.
:return: The modified Word document.
"""
- section = doc.add_section(
- start_type=WD_SECTION.NEW_PAGE
- ) # Add a new section for a new page
- section.left_margin = section.right_margin = Pt(72) # 1-inch margins
- section.top_margin = section.bottom_margin = Pt(72)
+ section = doc.add_section(start_type=WD_SECTION.NEW_PAGE)
+ # Set A4 paper dimensions
+ section.page_width = Mm(210)
+ section.page_height = Mm(297)
+ section.left_margin = section.right_margin = Pt(54)
+ section.top_margin = section.bottom_margin = Pt(54)
usable_height = section.page_height - section.top_margin - section.bottom_margin
- line_spacing = Pt(18) # Approx. 1.5x line spacing for handwriting clarity
+ usable_width = section.page_width - section.left_margin - section.right_margin
+ line_spacing = Pt(24) # ~8.5mm college-ruled spacing
num_lines = int(usable_height / line_spacing)
- # Add a single paragraph with blank lines separated by line breaks
lined_paragraph = doc.add_paragraph()
- lined_paragraph.alignment = WD_PARAGRAPH_ALIGNMENT.LEFT
lined_paragraph.paragraph_format.space_before = Pt(0)
lined_paragraph.paragraph_format.space_after = Pt(0)
lined_paragraph.paragraph_format.line_spacing = line_spacing
- for _ in range(num_lines - 3):
- lined_paragraph.add_run("_" * 100) # Add a visible placeholder for each line
- lined_paragraph.add_run("\n") # Add a line break to simulate ruled lines
+ # Add a right-aligned tab stop set at the exact right margin edge with a bottom line leader
+ lined_paragraph.paragraph_format.tab_stops.add_tab_stop(
+ usable_width, WD_TAB_ALIGNMENT.RIGHT, WD_TAB_LEADER.LINES
+ )
+ # Insert a tab character for each line to extend the rule to the right margin
+ for i in range(num_lines - 1):
+ lined_paragraph.add_run("\t")
+ if i < num_lines - 2:
+ lined_paragraph.add_run("\n")
return doc
diff --git a/backend/templates/html/header_enclosing.html b/backend/templates/html/header_enclosing.html
index 049d0a68..f06a66ce 100644
--- a/backend/templates/html/header_enclosing.html
+++ b/backend/templates/html/header_enclosing.html
@@ -83,6 +83,18 @@
font-family: var(--universal-font);
}
+ a {
+ color: #000000; /* Makes the text black */
+ text-decoration: underline; /* Ensures the underline is visible */
+ background-color: #e0e0e0; /* Adds a light grey background highlight */
+ padding: 2px 4px; /* Optional: Adds a little breathing room around the highlight */
+ border-radius: 3px; /* Optional: Softens the edges of the highlight */
+ }
+
+ h1 {
+ text-align: center;
+ }
+
/* Makes list of translation words left justified after an h2 (the resource type name: Translation Words) */
h2 + ul {
margin-left: 0;
diff --git a/frontend/.env b/frontend/.env
index f6d3fed6..6274ab66 100644
--- a/frontend/.env
+++ b/frontend/.env
@@ -31,3 +31,6 @@ PUBLIC_STET_TARGET_LANG_CODES_NAMES_URL = '/stet/target_languages'
PUBLIC_PASSAGES_URL = '/passages/document_docx'
PUBLIC_CHAPTERS_IN_BOOKS_URL = '/chapters_in_books'
PUBLIC_PRODUCTION_DOMAIN = 'bibleineverylanguage.org'
+
+PUBLIC_TURN_OFF_EPUB = true
+PUBLIC_TURN_OFF_PDF = true
diff --git a/frontend/src/lib/stores/SettingsStore.ts b/frontend/src/lib/stores/SettingsStore.ts
index 09c9329f..0bbc9c5d 100644
--- a/frontend/src/lib/stores/SettingsStore.ts
+++ b/frontend/src/lib/stores/SettingsStore.ts
@@ -8,7 +8,7 @@ const chunkSizeDefault: string = PUBLIC_CHUNK_SIZE_CHAPTER
export let layoutForPrintStore: Writable = writable(false)
export let assemblyStrategyKindStore: Writable = writable(groupingOrderDefault)
export let assemblyStrategyChunkSizeStore: Writable = writable(chunkSizeDefault)
-export let docTypeStore: Writable = writable('pdf')
+export let docTypeStore: Writable = writable('docx')
export let generatePdfStore: Writable = writable(true)
export let generateEpubStore: Writable = writable(false)
export let generateDocxStore: Writable = writable(false)
diff --git a/frontend/src/routes/settings/+page.svelte b/frontend/src/routes/settings/+page.svelte
index 5070ca92..d6891609 100644
--- a/frontend/src/routes/settings/+page.svelte
+++ b/frontend/src/routes/settings/+page.svelte
@@ -1,5 +1,6 @@