Skip to content

fix: support CJK title and author identity matching #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xzjncu wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: support CJK title and author identity matching #16

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions skills/deeppapernote/scripts/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,10 @@ def strip_tags(text: str) -> str:


def normalize_title(text: str) -> str:
return re.sub(r"[^a-z0-9\s]", "", normalize_whitespace(text).lower()).strip()
normalized = unicodedata.normalize("NFKC", normalize_whitespace(text)).casefold()
return normalize_whitespace("".join(
character for character in normalized if character.isalnum() or character.isspace()
))


LOCAL_PDF_PREFIX_PATTERN = re.compile(r"^(?:[^-]{1,120})\s+-\s+(?:19|20)\d{2}\s+-\s+")
Expand Down Expand Up @@ -576,6 +579,10 @@ def _full_leading_author_matches(
work_authors = _dedupe_string_list(work_record.get("authors", []))
if not source_authors or not work_authors:
return False
source_normalized = unicodedata.normalize("NFKC", source_authors[0]).casefold()
work_normalized = unicodedata.normalize("NFKC", work_authors[0]).casefold()
if source_normalized == work_normalized:
return True
source_parts = _author_identity_parts(source_authors[0])
work_parts = _author_identity_parts(work_authors[0])
if len(source_parts) < 2 or len(work_parts) < 2:
Expand All @@ -598,9 +605,11 @@ def _leading_author_status(
work_authors = _dedupe_string_list(work_record.get("authors", []))
if not source_authors or not work_authors:
return None
source_normalized = unicodedata.normalize("NFKC", source_authors[0]).casefold()
work_normalized = unicodedata.normalize("NFKC", work_authors[0]).casefold()
source_key = _author_key(source_authors[0])
work_key = _author_key(work_authors[0])
status = "match" if source_key and source_key == work_key else "conflict"
status = "match" if source_normalized == work_normalized or (source_key and source_key == work_key) else "conflict"
return {
"kind": "leading_author",
"status": status,
Expand Down Expand Up @@ -3990,4 +3999,4 @@ def pick_sentences_by_keywords(text: str, keywords: list[str], *, limit: int = 5
picked.append(normalize_whitespace(sentence))
if len(picked) >= limit:
break
return picked
return picked