Tokenizzazione reale e mappe linguaggi ampliate

README.md

@@ -16,6 +16,9 @@
 - Passwords and API keys in the output are masked for safety
 - Exports larger than the chosen token limit are automatically split into multiple files
 - Select a token limit preset (ChatGPT, Gemini, Claude) or set a custom value
+- Token counting usa i tokenizer ufficiali (tiktoken, anthropic, Google) quando disponibili
+- Scegli la lingua dell'interfaccia tramite un menu a discesa: le opzioni vengono rilevate automaticamente dai file JSON in `promptpack/locales`
+- Le stringhe tradotte sono raccolte in file JSON dentro `promptpack/locales` per facilitare l'aggiunta di nuove lingue
 - Copy the preview to the clipboard with one click
 - Select or deselect all files at once when choosing what to include
 - Remaining token counter shows usage versus limit

promptpack/i18n.py

@@ -0,0 +1,29 @@
+"""Utility per la gestione delle traduzioni."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+LOCALES_DIR = Path(__file__).parent / "locales"
+
+_cache: dict[str, dict[str, str]] = {}
+
+
+def load_translations(lang: str) -> dict[str, str]:
+    """Carica le stringhe localizzate dal file corrispondente."""
+    if lang not in _cache:
+        path = LOCALES_DIR / f"{lang}.json"
+        if not path.exists():
+            path = LOCALES_DIR / "eng.json"
+        try:
+            with path.open("r", encoding="utf-8") as f:
+                _cache[lang] = json.load(f)
+        except Exception:
+            _cache[lang] = {}
+    return _cache[lang]
+
+
+def available_languages() -> list[str]:
+    """Ritorna l'elenco dei codici lingua disponibili."""
+    return sorted(p.stem for p in LOCALES_DIR.glob("*.json"))

promptpack/locales/eng.json

@@ -0,0 +1,51 @@
+{
+    "source": "Source",
+    "source_folder": "Source Folder",
+    "browse": "Browse",
+    "select_files": "Select Files",
+    "preview": "Preview",
+    "copy": "Copy to Clipboard",
+    "browser": "Preview in browser",
+    "live_preview": "Live Preview",
+    "output": "Output",
+    "dest_folder": "Destination Folder",
+    "generate": "Generate",
+    "settings": "\u2699\ufe0f Settings",
+    "settings_title": "Settings",
+    "default_selection": "Default Selection",
+    "allowed_exts": "Default Allowed Extensions",
+    "excluded_dirs": "Default Excluded Directories",
+    "excluded_files": "Default Excluded Files",
+    "output_opts": "Output Options",
+    "export_format": "Export format:",
+    "include_headings": "Include File Headings",
+    "use_code": "Use Code Blocks",
+    "tree_only": "Tree only",
+    "token_limit": "Token limit",
+    "theme": "Theme",
+    "light": "Light",
+    "dark": "Dark",
+    "language": "Language",
+    "english": "English",
+    "italian": "Italiano",
+    "save": "Save",
+    "select_files_title": "Select Files to Include",
+    "select_deselect": "Select/Deselect All",
+    "confirm": "Confirm Selection",
+    "no_files": "No Files",
+    "no_preview": "No files selected for preview.",
+    "copied": "Copied",
+    "content_copied": "Content copied to clipboard",
+    "preview_copied": "Preview copied to clipboard",
+    "error": "Error",
+    "select_source_first": "Please select the source folder first",
+    "limit_reached": "Limit reached",
+    "limit_msg": "Reached the limit of {max} tokens. Some files were skipped.",
+    "done": "Done",
+    "files_generated": "Files generated:\n{msg}",
+    "no_selected": "No files selected",
+    "need_folders": "Please select both source and destination folders",
+    "token_header": "Token estimate: {count} (remaining {remaining})\n{sep}\n",
+    "tokens": "Tokens: {tokens} / {max}",
+    "project_label": "Project: {name} - {date}"
+}

promptpack/locales/it.json

@@ -0,0 +1,51 @@
+{
+    "source": "Origine",
+    "source_folder": "Cartella sorgente",
+    "browse": "Sfoglia",
+    "select_files": "Seleziona file",
+    "preview": "Anteprima",
+    "copy": "Copia negli appunti",
+    "browser": "Anteprima nel browser",
+    "live_preview": "Anteprima live",
+    "output": "Output",
+    "dest_folder": "Cartella di destinazione",
+    "generate": "Genera",
+    "settings": "\u2699\ufe0f Impostazioni",
+    "settings_title": "Impostazioni",
+    "default_selection": "Selezione predefinita",
+    "allowed_exts": "Estensioni consentite predefinite",
+    "excluded_dirs": "Cartelle escluse predefinite",
+    "excluded_files": "File esclusi predefiniti",
+    "output_opts": "Opzioni di output",
+    "export_format": "Formato di esportazione:",
+    "include_headings": "Includi intestazioni dei file",
+    "use_code": "Usa blocchi di codice",
+    "tree_only": "Solo struttura",
+    "token_limit": "Limite token",
+    "theme": "Tema",
+    "light": "Chiaro",
+    "dark": "Scuro",
+    "language": "Lingua",
+    "english": "Inglese",
+    "italian": "Italiano",
+    "save": "Salva",
+    "select_files_title": "Seleziona i file da includere",
+    "select_deselect": "Seleziona/Deseleziona tutto",
+    "confirm": "Conferma selezione",
+    "no_files": "Nessun file",
+    "no_preview": "Nessun file selezionato per l'anteprima.",
+    "copied": "Copiato",
+    "content_copied": "Contenuto copiato negli appunti",
+    "preview_copied": "Anteprima copiata negli appunti",
+    "error": "Errore",
+    "select_source_first": "Seleziona prima la cartella sorgente",
+    "limit_reached": "Limite raggiunto",
+    "limit_msg": "Raggiunto il limite di {max} token. Alcuni file sono stati saltati.",
+    "done": "Fatto",
+    "files_generated": "File generati:\n{msg}",
+    "no_selected": "Nessun file selezionato",
+    "need_folders": "Seleziona sia la cartella sorgente che quella di destinazione",
+    "token_header": "Stima token: {count} (restano {remaining})\n{sep}\n",
+    "tokens": "Token: {tokens} / {max}",
+    "project_label": "Progetto: {name} - {date}"
+}

promptpack/settings.py

@@ -14,6 +14,7 @@
     "include_heading": True,
     "use_code_block": True,
     "theme": "dark",
+    "language": "eng",
     # Maximum tokens per preview or export file
     "max_tokens": 200_000,
     # Remember last source folder and selected files
@@ -36,6 +37,8 @@ def load_settings():
                     data["last_start_folder"] = DEFAULT_SETTINGS["last_start_folder"]
                 if "last_selected_files" not in data:
                     data["last_selected_files"] = DEFAULT_SETTINGS["last_selected_files"]
+                if "language" not in data:
+                    data["language"] = DEFAULT_SETTINGS["language"]
                 return {**DEFAULT_SETTINGS, **data}
         except Exception:
             return DEFAULT_SETTINGS.copy()

promptpack/tokenizer.py

@@ -0,0 +1,60 @@
+"""Utility functions for token counting with real tokenization models."""
+
+from __future__ import annotations
+
+try:
+    import tiktoken  # type: ignore
+except Exception:  # pragma: no cover - optional dependency
+    tiktoken = None
+
+try:
+    from anthropic import Anthropic  # type: ignore
+except Exception:  # pragma: no cover - optional dependency
+    Anthropic = None
+
+try:
+    import google.generativeai as genai  # type: ignore
+except Exception:  # pragma: no cover - optional dependency
+    genai = None
+
+
+def gpt_tokens(text: str) -> int:
+    """Count tokens using OpenAI tiktoken if available."""
+    if tiktoken is None:
+        # Fallback semplice se la libreria non è disponibile
+        return len(text) // 4
+    enc = tiktoken.get_encoding("cl100k_base")
+    return len(enc.encode(text))
+
+
+def claude_tokens(text: str) -> int:
+    """Count tokens using Anthropic's official tokenizer if available."""
+    if Anthropic is None:
+        return len(text) // 4
+    client = Anthropic()
+    return client.count_tokens(text)
+
+
+def gemini_tokens(text: str) -> int:
+    """Count tokens using Google generative AI library if available."""
+    if genai is None:
+        return len(text) // 4
+    info = genai.token_count(text)
+    # google-generativeai returns a dict with 'token_count'
+    return info.get("token_count", len(info.get("tokens", [])))
+
+
+MODEL_DISPATCH = {
+    "gpt": gpt_tokens,
+    "claude": claude_tokens,
+    "gemini": gemini_tokens,
+}
+
+
+def estimate_token_count(text: str, model: str = "gpt") -> int:
+    """Return number of tokens for the chosen model."""
+    func = MODEL_DISPATCH.get(model)
+    if func is None:
+        raise ValueError(f"Modello sconosciuto: {model}")
+    return func(text)
+

promptpack/utils.py

@@ -7,13 +7,33 @@
 import webbrowser
 import markdown
 
+from .tokenizer import estimate_token_count
+
 LANG_MAP = {
     ".py": "python",
     ".js": "javascript",
     ".ts": "typescript",
     ".php": "php",
     ".html": "html",
     ".css": "css",
+    ".sh": "bash",
+    ".json": "json",
+    ".yml": "yaml",
+    ".yaml": "yaml",
+    ".java": "java",
+    ".c": "c",
+    ".cpp": "cpp",
+    ".h": "c",
+    ".cs": "csharp",
+    ".go": "go",
+    ".rs": "rust",
+    ".kt": "kotlin",
+    ".swift": "swift",
+    ".rb": "ruby",
+    ".md": "markdown",
+    ".txt": "",
+    ".it": "",
+    ".en": "",
 }
 
 
@@ -26,8 +46,6 @@ def apply_icon(window):
         print(f"Icon not loaded: {e}")
 
 
-def estimate_token_count(text: str) -> int:
-    return int(len(text) / 4)
 
 
 SENSITIVE_PATTERNS = [

promptpack_settings.json

@@ -21,6 +21,7 @@
     "include_heading": true,
     "use_code_block": true,
     "theme": "dark",
+    "language": "eng",
     "max_tokens": 200000,
     "last_start_folder": "",
     "last_selected_files": []