Expand core language packs to 500 words

README.md

@@ -33,7 +33,7 @@ deployment automation.
   `initData`, never creates learning or billing records on open, and saves swipe
   answers through protected APIs using the same deterministic SRS as the bot.
 - A separate browser dictionary at `/dictionary/`: seven reviewed starter
-  packs (700 entries), bilingual local search, saved words, short written
+  packs (3,500 entries), bilingual local search, saved words, short written
   practice, explicit offline storage, standalone HTML download and saved-word
   CSV export. It opens without Telegram or a learner account. Words outside
   these packs can be sent to Yandex Translate by an explicitly labelled link.
@@ -130,6 +130,9 @@ release and Cloudflare route deliberately use separate runbooks and gates.
 ## License
 
 The application source code is MIT-licensed; see [`LICENSE`](LICENSE). The
-seven generated schema-v2 starter packs derive from the project's original
-`content/basic_100.tsv` matrix. Verify provenance and redistribution rights
-before republishing legacy or externally supplied vocabulary datasets.
+seven generated schema-v2 starter packs retain the project's original first
+100 aligned rows and add 400 curated rows per language from open,
+Wiktionary-derived sources. Japanese retains its original first 100 entries and
+adds 400 reviewed entries. See [`docs/basic-language-packs.md`](docs/basic-language-packs.md)
+for exact source revisions, attribution, third-party license notes, and the
+reason the legacy `*-basics-100` IDs and `basic_100.tsv` filename remain.

content/catalog.json

@@ -51,17 +51,17 @@
       "direction": "ltr",
       "flag": "🇬🇧",
       "meaning_flag": "🇷🇺",
-      "label": "🇬🇧 English · 100",
-      "title": "Английский: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с русскими значениями, IPA и озвучкой.",
+      "label": "🇬🇧 English · 500",
+      "title": "Английский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с русскими значениями, IPA и озвучкой.",
       "filename": "words_en_basic.json",
       "storage_key": "en_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "ipa",
         "transcription_position": "after",
@@ -103,17 +103,17 @@
       "direction": "ltr",
       "flag": "🇫🇷",
       "meaning_flag": "🇷🇺",
-      "label": "🇫🇷 Français · 100",
-      "title": "Французский: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с русскими значениями, IPA и озвучкой.",
+      "label": "🇫🇷 Français · 500",
+      "title": "Французский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с русскими значениями, IPA и озвучкой.",
       "filename": "words_fr_basic.json",
       "storage_key": "fr_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "ipa",
         "transcription_position": "after",
@@ -129,17 +129,17 @@
       "direction": "ltr",
       "flag": "🇩🇪",
       "meaning_flag": "🇷🇺",
-      "label": "🇩🇪 Deutsch · 100",
-      "title": "Немецкий: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с русскими значениями, IPA и озвучкой.",
+      "label": "🇩🇪 Deutsch · 500",
+      "title": "Немецкий: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с русскими значениями, IPA и озвучкой.",
       "filename": "words_de_basic.json",
       "storage_key": "de_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "ipa",
         "transcription_position": "after",
@@ -155,17 +155,17 @@
       "direction": "ltr",
       "flag": "🇯🇵",
       "meaning_flag": "🇷🇺",
-      "label": "🇯🇵 日本語",
-      "title": "Японский: 100 базовых слов",
-      "description": "Базовый набор: русский смысл, иероглифы, ромадзи и произношение.",
+      "label": "🇯🇵 日本語 · 500",
+      "title": "Японский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов: русский смысл, японское написание, чтение, ромадзи и озвучка.",
       "filename": "words_ja.json",
       "storage_key": "ja",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 1,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "hepburn",
         "transcription_position": "before",
@@ -181,17 +181,17 @@
       "direction": "rtl",
       "flag": "🇸🇦",
       "meaning_flag": "🇷🇺",
-      "label": "🇸🇦 العربية · 100",
-      "title": "Арабский: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с русскими значениями, латиницей и озвучкой.",
+      "label": "🇸🇦 العربية · 500",
+      "title": "Арабский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с русскими значениями, латиницей и озвучкой.",
       "filename": "words_ar_basic.json",
       "storage_key": "ar_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "learner-latin",
         "transcription_position": "before",
@@ -207,17 +207,17 @@
       "direction": "ltr",
       "flag": "🇨🇳",
       "meaning_flag": "🇷🇺",
-      "label": "🇨🇳 中文 · 100",
-      "title": "Китайский: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с русскими значениями, пиньинем и озвучкой.",
+      "label": "🇨🇳 中文 · 500",
+      "title": "Китайский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с русскими значениями, пиньинем и озвучкой.",
       "filename": "words_zh_basic.json",
       "storage_key": "zh_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "pinyin",
         "transcription_position": "before",
@@ -233,17 +233,17 @@
       "direction": "ltr",
       "flag": "🇷🇺",
       "meaning_flag": "🇷🇺",
-      "label": "🇷🇺 Русский · 100",
-      "title": "Русский: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с простыми определениями, латиницей и озвучкой.",
+      "label": "🇷🇺 Русский · 500",
+      "title": "Русский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с простыми определениями, латиницей и озвучкой.",
       "filename": "words_ru_basic.json",
       "storage_key": "ru_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "learner-latin",
         "transcription_position": "before",
@@ -259,17 +259,17 @@
       "direction": "ltr",
       "flag": "🇪🇸",
       "meaning_flag": "🇷🇺",
-      "label": "🇪🇸 Español · 100",
-      "title": "Испанский: 100 базовых слов",
-      "description": "Бесплатный стартовый набор с русскими значениями, IPA и озвучкой.",
+      "label": "🇪🇸 Español · 500",
+      "title": "Испанский: 500 базовых слов",
+      "description": "Бесплатный базовый набор из 500 слов с русскими значениями, IPA и озвучкой.",
       "filename": "words_es_basic.json",
       "storage_key": "es_basic",
       "visibility": "public",
       "is_free": true,
       "status": "published",
       "content_schema": 2,
-      "content_version": 1,
-      "entry_count": 100,
+      "content_version": 2,
+      "entry_count": 500,
       "pronunciation": {
         "transcription_system": "ipa",
         "transcription_position": "after",

docs/basic-language-packs.md

@@ -1,6 +1,6 @@
 # Eight basic language packs
 
-MY DICTIONARY ships free 100-entry starter packs for English, French, German,
+MY DICTIONARY ships free 500-entry starter packs for English, French, German,
 Japanese, Arabic, Chinese, Russian, and Spanish. The existing Vietnamese pack
 remains available as an additional free pack.
 
@@ -9,7 +9,7 @@ remains available as an additional free pack.
 - Every pack provides a target form, a Russian meaning, a Latin transcription,
   a speech form, and at least one learning topic.
 - The seven new schema v2 packs share stable entry IDs and the same ten topics,
-  with ten entries in each topic.
+  with 50 entries in each topic.
 - Russian uses short Russian definitions as meanings so target and meaning are
   not duplicates.
 - IPA is used for English, French, German, and Spanish. Chinese uses pinyin;
@@ -20,9 +20,33 @@ remains available as an additional free pack.
 
 ## Source and generation
 
-`content/basic_100.tsv` is the aligned source of truth for the seven new packs.
-Its vocabulary matrix is original project content rather than a copied export
-from a third-party learning service.
+`content/basic_100.tsv` is the aligned source of truth for the seven schema v2
+packs. Its legacy filename is retained because deployment allowlists and older
+automation refer to it. The first 100 rows are the original project matrix;
+the 400-row extension was curated from the openly available
+[`appsinacup/polyglot-dictionaries`](https://github.com/appsinacup/polyglot-dictionaries)
+1000-word lists at commit `0375cbb7117c8833b8d69da5e8c60a052bab0b28`, then
+reviewed and aligned to one everyday sense per entry. That repository is MIT
+licensed and identifies Wiktionary as the source of its translations;
+Wiktionary-derived content remains subject to Wiktionary attribution and
+CC BY-SA terms.
+
+English, French, German, and Spanish IPA was drawn from
+[`open-dict-data/ipa-dict`](https://github.com/open-dict-data/ipa-dict) at
+commit `43c3570eb3553bdd19fccd2bd0091534889af023`. Missing headwords were checked
+against Wiktionary-derived WiktAPI data, and a small number of multiword forms
+were composed from the same word-level IPA. `ipa-dict` is MIT unless a credited
+upstream dataset states another license; notably its German data is
+Wiktionary-derived CC BY-SA. Pinyin and Hepburn readings were generated with
+`pypinyin` 0.55.0 and `pykakasi` 2.3.0 respectively. The reviewed, normalized
+outputs are checked into this repository; builds do not fetch these services or
+packages.
+
+The historical pack IDs such as `en-basics-100`, the storage keys, and the
+source filename are intentionally unchanged. They are persistence identities,
+not counts: changing them would split existing learner progress. Catalog labels
+and `entry_count` are the authoritative UI metadata and now report 500.
+
 Run the deterministic generator after changing it:
 
 ```sh
@@ -34,6 +58,8 @@ The second command is suitable for CI and fails if a generated JSON file is
 missing or differs from the source matrix.
 
 The Japanese pack intentionally stays on the schema v1 compatibility adapter.
-This preserves its existing example sentences and historical learner progress
-identities while still exposing canonical target, meaning, romaji, topics, and
-speech fields to the bot.
+Its original first 100 entries and order are unchanged; 400 reviewed entries
+were appended with kana reading, ASCII Hepburn romaji and topic metadata. This
+preserves existing example sentences and historical learner progress identities
+while exposing canonical target, meaning, romaji, topics, and speech fields to
+the bot.

docs/offline-dictionary.md

@@ -7,10 +7,10 @@ Telegram account and does not read learner storage.
 
 ## Capabilities and boundaries
 
-- 700 entries from seven explicitly redistribution-approved public, free,
+- 3,500 entries from seven explicitly redistribution-approved public, free,
   published schema-v2 starter packs: English, French, German, Arabic, Chinese,
-  Russian and Spanish. These represent 100 aligned vocabulary concepts, not
-  700 distinct words in each language. Private/admin packs and the legacy
+  Russian and Spanish. These represent 500 aligned vocabulary concepts, not
+  3,500 distinct words in each language. Private/admin packs and the legacy
   Japanese/Vietnamese packs are not exported.
 - Target/translation language selectors, pair swapping, Unicode-normalized
   bidirectional search, accepted Russian meanings and transcription.

scripts/build_basic_packs.py

@@ -28,6 +28,31 @@
     "actions",
     "descriptions",
 )
+EXPECTED_SOURCE_ROWS = 500
+ENTRIES_PER_TOPIC = 50
+
+# These polysemous concepts are intentionally pinned to one everyday learning
+# sense across every generated language.  Keeping the checks beside the source
+# builder prevents a future upstream refresh from silently mixing unrelated
+# senses (for example, ``driver`` as an animal handler or ``capital`` as money).
+EXPECTED_ALIGNED_SENSES = {
+    "call": {"en": "call", "fr": "appel", "de": "Anruf", "ar": "اِتِّصَال", "zh": "呼叫", "ru": "звоно́к", "es": "llamada"},
+    "capital": {"en": "capital", "fr": "capitale", "de": "Hauptstadt", "ar": "عَاصِمَة", "zh": "首都", "ru": "столи́ца", "es": "capital"},
+    "card": {"en": "card", "fr": "carte", "de": "Karte", "ar": "بِطَاقَة", "zh": "卡片", "ru": "ка́рточка", "es": "tarjeta"},
+    "driver": {"en": "driver", "fr": "conducteur", "de": "Fahrer", "ar": "سَائِق", "zh": "司机", "ru": "води́тель", "es": "conductor"},
+    "let": {"en": "let", "fr": "permettre", "de": "erlauben", "ar": "سَمَحَ", "zh": "让", "ru": "позволя́ть", "es": "permitir"},
+    "light": {"en": "light", "fr": "lumière", "de": "Licht", "ar": "ضَوْء", "zh": "光", "ru": "свет", "es": "luz"},
+    "miss": {"en": "miss", "fr": "manquer", "de": "vermissen", "ar": "اِشْتَاقَ إِلَى", "zh": "想念", "ru": "скуча́ть по", "es": "extrañar"},
+    "news": {"en": "news", "fr": "nouvelles", "de": "Neuigkeiten", "ar": "أَخْبَار", "zh": "消息", "ru": "но́вости", "es": "noticias"},
+    "note": {"en": "note", "fr": "note", "de": "Notiz", "ar": "مُلَاحَظَة", "zh": "字条", "ru": "заме́тка", "es": "nota"},
+    "point": {"en": "point", "fr": "point", "de": "Punkt", "ar": "نُقْطَة", "zh": "点", "ru": "то́чка", "es": "punto"},
+    "present": {"en": "present", "fr": "cadeau", "de": "Geschenk", "ar": "هَدِيَّة", "zh": "礼物", "ru": "пода́рок", "es": "regalo"},
+    "sentence": {"en": "sentence", "fr": "phrase", "de": "Satz", "ar": "جُمْلَة", "zh": "句子", "ru": "предложе́ние", "es": "oración"},
+    "spell": {"en": "spell", "fr": "épeler", "de": "buchstabieren", "ar": "هَجَّى", "zh": "拼写", "ru": "произноси́ть по бу́квам", "es": "deletrear"},
+    "spelling": {"en": "spelling", "fr": "orthographe", "de": "Rechtschreibung", "ar": "تَهْجِئَة", "zh": "拼写法", "ru": "правописа́ние", "es": "ortografía"},
+    "subject": {"en": "subject", "fr": "matière", "de": "Schulfach", "ar": "مَادَّة دِرَاسِيَّة", "zh": "学科", "ru": "уче́бный предме́т", "es": "asignatura"},
+    "watch": {"en": "watch", "fr": "montre", "de": "Uhr", "ar": "سَاعَة يَد", "zh": "手表", "ru": "нару́чные часы́", "es": "reloj"},
+}
 
 
 @dataclass(frozen=True)
@@ -107,8 +132,10 @@ def load_rows(path: Path = SOURCE) -> list[dict[str, str]]:
             raise SourceError(f"unexpected header in {path.relative_to(ROOT)}")
         rows = list(reader)
 
-    if len(rows) != 100:
-        raise SourceError(f"expected 100 source rows, found {len(rows)}")
+    if len(rows) != EXPECTED_SOURCE_ROWS:
+        raise SourceError(
+            f"expected {EXPECTED_SOURCE_ROWS} source rows, found {len(rows)}"
+        )
 
     entry_ids: set[str] = set()
     targets = {pack.language: set() for pack in PACKS}
@@ -132,15 +159,23 @@ def load_rows(path: Path = SOURCE) -> list[dict[str, str]]:
             target, _, _, _ = _parse_cell(
                 row[pack.language], row=number, language=pack.language
             )
+            expected = EXPECTED_ALIGNED_SENSES.get(entry_id, {}).get(pack.language)
+            if expected is not None and target != expected:
+                raise SourceError(
+                    f"row {number}: {entry_id}.{pack.language} must stay {expected!r}"
+                )
             target_key = target.casefold()
             if target_key in targets[pack.language]:
                 raise SourceError(
                     f"row {number}: duplicate {pack.language} target {target}"
                 )
             targets[pack.language].add(target_key)
 
-    if topic_counts != Counter({topic: 10 for topic in TOPICS}):
-        raise SourceError(f"expected ten entries per topic, found {dict(topic_counts)}")
+    if topic_counts != Counter({topic: ENTRIES_PER_TOPIC for topic in TOPICS}):
+        raise SourceError(
+            f"expected {ENTRIES_PER_TOPIC} entries per topic, "
+            f"found {dict(topic_counts)}"
+        )
     return rows
 
 

tests/browser/dictionary.cjs

@@ -53,7 +53,7 @@ const path = require("node:path");
       await page.locator("#target-language").selectOption(target);
       for (const native of codes.filter(code=>code!==target)) {
         await page.locator("#native-language").selectOption(native);
-        assert.match(await page.locator("#pack-note").innerText(),/100/);
+        assert.match(await page.locator("#pack-note").innerText(),/500/);
       }
     }
     await page.locator("#target-language").selectOption("fr");