diff --git a/.github/scripts/check_language_toml.py b/.github/scripts/check_language_toml.py index 2e2353584d..494f90962e 100644 --- a/.github/scripts/check_language_toml.py +++ b/.github/scripts/check_language_toml.py @@ -20,21 +20,8 @@ import os import argparse import re import json - -try: - import tomllib # Python 3.11+ -except ImportError: - try: - import toml as tomllib_fallback - tomllib = None - except ImportError: - tomllib = None - tomllib_fallback = None - -try: - import tomli_w # For writing TOML files -except ImportError: - tomli_w = None +import tomllib # Python 3.11+ (stdlib) +import tomli_w # For writing TOML files def find_duplicate_keys(file_path, keys=None, prefix=""): @@ -51,14 +38,8 @@ def find_duplicate_keys(file_path, keys=None, prefix=""): duplicates = [] # Load TOML file - if tomllib: - with open(file_path, 'rb') as file: - data = tomllib.load(file) - elif tomllib_fallback: - with open(file_path, 'r', encoding='utf-8') as file: - data = tomllib_fallback.load(file) - else: - return [] # Cannot check without TOML support + with open(file_path, 'rb') as file: + data = tomllib.load(file) def process_dict(obj, current_prefix=""): for key, value in obj.items(): @@ -86,14 +67,8 @@ def parse_toml_file(file_path): :param file_path: Path to the TOML file. :return: Dictionary with flattened keys. """ - if tomllib: - with open(file_path, 'rb') as file: - data = tomllib.load(file) - elif tomllib_fallback: - with open(file_path, 'r', encoding='utf-8') as file: - data = tomllib_fallback.load(file) - else: - raise RuntimeError("TOML support not available. Install 'toml' or upgrade to Python 3.11+") + with open(file_path, 'rb') as file: + data = tomllib.load(file) def flatten_dict(d, parent_key="", sep="."): items = {} @@ -135,14 +110,8 @@ def write_toml_file(file_path, updated_properties): """ nested_data = unflatten_dict(updated_properties) - if tomli_w: - with open(file_path, "wb") as file: - tomli_w.dump(nested_data, file) - elif tomllib_fallback and hasattr(tomllib_fallback, 'dump'): - with open(file_path, "w", encoding="utf-8", newline="\n") as file: - tomllib_fallback.dump(nested_data, file) - else: - raise RuntimeError("TOML writing not supported. Install 'tomli-w' library") + with open(file_path, "wb") as file: + tomli_w.dump(nested_data, file) def update_missing_keys(reference_file, file_list, branch=""): diff --git a/.github/workflows/check_toml.yml b/.github/workflows/check_toml.yml index 63f8da27ee..8fc1a75b8b 100644 --- a/.github/workflows/check_toml.yml +++ b/.github/workflows/check_toml.yml @@ -1,9 +1,6 @@ -name: Check TOML Translation Files on PR (V3) +name: Check TOML Translation Files on PR -# This workflow validates TOML translation files for V3 format -# For legacy formats, see: -# - check_properties.yml (V1 backend .properties) -# - Legacy JSON validation (V2, deprecated) +# This workflow validates TOML translation files on: pull_request_target: @@ -203,7 +200,7 @@ jobs: - name: Install Python dependencies run: | - pip install toml tomli-w + pip install tomli-w - name: Run Python script to check files id: run-check diff --git a/.github/workflows/sync_files_v2.yml b/.github/workflows/sync_files_v2.yml index 4b33cd2c39..935252be25 100644 --- a/.github/workflows/sync_files_v2.yml +++ b/.github/workflows/sync_files_v2.yml @@ -1,4 +1,4 @@ -name: Sync Files V3 (TOML) +name: Sync Files (TOML) on: workflow_dispatch: @@ -54,7 +54,7 @@ jobs: - name: Install Python dependencies run: | - pip install toml tomli-w + pip install tomli-w - name: Sync translation TOML files run: | @@ -88,17 +88,17 @@ jobs: signoff: true branch: sync_readme_v3 base: main - title: ":globe_with_meridians: [V3] Sync Translations + Update README Progress Table" + title: ":globe_with_meridians: Sync Translations + Update README Progress Table" body: | ### Description of Changes - This Pull Request was automatically generated to synchronize updates to translation files and documentation for **V3 (TOML format)**. Below are the details of the changes made: + This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made: #### **1. Synchronization of Translation Files** - Updated translation files (`frontend/public/locales/*/translation.toml`) to reflect changes in the reference file `en-GB/translation.toml`. - Ensured consistency and synchronization across all supported language files. - Highlighted any missing or incomplete translations. - - **Format**: TOML (V3, current) - migrated from JSON (V2) and .properties (V1) + - **Format**: TOML #### **2. Update README.md** - Generated the translation progress table in `README.md` using `counter_translation_v3.py`. diff --git a/ADDING_TOOLS.md b/ADDING_TOOLS.md index ef1501bfc9..d24641b000 100644 --- a/ADDING_TOOLS.md +++ b/ADDING_TOOLS.md @@ -202,10 +202,10 @@ const [ToolName] = (props: BaseToolProps) => { ## 5. Add Translations Update translation files. **Important: Only update `en-GB` files** - other languages are handled separately. -**File to update:** `frontend/public/locales/en-GB/translation.json` +**File to update:** `frontend/public/locales/en-GB/translation.toml` **Required Translation Keys**: -```json +```toml { "home": { "[toolName]": { @@ -251,7 +251,7 @@ Update translation files. **Important: Only update `en-GB` files** - other langu ``` **Translation Notes:** -- **Only update `en-GB/translation.json`** - other locale files are managed separately +- **Only update `en-GB/translation.toml`** - other locale files are managed separately - Use descriptive keys that match your component's `t()` calls - Include tooltip translations if you created tooltip hooks - Add `options.*` keys if your tool has settings with descriptions diff --git a/scripts/translations/ai_translation_helper.py b/scripts/translations/ai_translation_helper.py index 2a34a8d521..8e5dbe35dc 100644 --- a/scripts/translations/ai_translation_helper.py +++ b/scripts/translations/ai_translation_helper.py @@ -15,21 +15,8 @@ import argparse import re from datetime import datetime import csv - -try: - import tomllib # Python 3.11+ -except ImportError: - try: - import toml as tomllib_fallback - tomllib = None - except ImportError: - tomllib = None - tomllib_fallback = None - -try: - import tomli_w # For writing TOML -except ImportError: - tomli_w = None +import tomllib # Python 3.11+ (stdlib) +import tomli_w # For writing TOML class AITranslationHelper: @@ -55,15 +42,8 @@ class AITranslationHelper: """Load TOML or JSON translation file based on extension.""" try: if file_path.suffix == '.toml': - if tomllib: - with open(file_path, 'rb') as f: - return tomllib.load(f) - elif tomllib_fallback: - with open(file_path, 'r', encoding='utf-8') as f: - return tomllib_fallback.load(f) - else: - print(f"Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+") - sys.exit(1) + with open(file_path, 'rb') as f: + return tomllib.load(f) else: # JSON with open(file_path, 'r', encoding='utf-8') as f: return json.load(f) @@ -74,12 +54,8 @@ class AITranslationHelper: def _save_translation_file(self, data: Dict, file_path: Path) -> None: """Save translation file (TOML or JSON) based on extension.""" if file_path.suffix == '.toml': - if tomli_w: - with open(file_path, 'wb') as f: - tomli_w.dump(data, f) - else: - print(f"Error: TOML writing not available. Install 'tomli_w'") - sys.exit(1) + with open(file_path, 'wb') as f: + tomli_w.dump(data, f) else: # JSON with open(file_path, 'w', encoding='utf-8') as f: json.dump(data, f, indent=2, ensure_ascii=False) @@ -106,10 +82,18 @@ class AITranslationHelper: for lang in languages: lang_dir = self.locales_dir / lang - try: - lang_file = self._find_translation_file(lang_dir) + + # Check if translation file exists, if not create empty structure + toml_file = lang_dir / "translation.toml" + json_file = lang_dir / "translation.json" + + if toml_file.exists(): + lang_file = toml_file lang_data = self._load_translation_file(lang_file) - except SystemExit: + elif json_file.exists(): + lang_file = json_file + lang_data = self._load_translation_file(lang_file) + else: # No translation file found, create empty structure lang_data = {} @@ -285,10 +269,16 @@ class AITranslationHelper: lang_dir = self.locales_dir / lang # Load existing data or create new - try: - lang_file = self._find_translation_file(lang_dir) + toml_file = lang_dir / "translation.toml" + json_file = lang_dir / "translation.json" + + if toml_file.exists(): + lang_file = toml_file lang_data = self._load_translation_file(lang_file) - except SystemExit: + elif json_file.exists(): + lang_file = json_file + lang_data = self._load_translation_file(lang_file) + else: # No translation file found, create new JSON file lang_data = {} lang_dir.mkdir(parents=True, exist_ok=True) @@ -347,15 +337,26 @@ class AITranslationHelper: for lang in languages: lang_dir = self.locales_dir / lang - try: - lang_file = self._find_translation_file(lang_dir) + toml_file = lang_dir / "translation.toml" + json_file = lang_dir / "translation.json" + + if toml_file.exists(): + lang_file = toml_file lang_data = self._load_translation_file(lang_file) lang_flat = self._flatten_dict(lang_data) value = lang_flat.get(key, '') if value.startswith('[UNTRANSLATED]'): value = '' row[lang] = value - except SystemExit: + elif json_file.exists(): + lang_file = json_file + lang_data = self._load_translation_file(lang_file) + lang_flat = self._flatten_dict(lang_data) + value = lang_flat.get(key, '') + if value.startswith('[UNTRANSLATED]'): + value = '' + row[lang] = value + else: row[lang] = '' writer.writerow(row) @@ -377,16 +378,25 @@ class AITranslationHelper: for lang in languages: lang_dir = self.locales_dir / lang - try: - lang_file = self._find_translation_file(lang_dir) + toml_file = lang_dir / "translation.toml" + json_file = lang_dir / "translation.json" + + if toml_file.exists(): + lang_file = toml_file + lang_data = self._load_translation_file(lang_file) + lang_flat = self._flatten_dict(lang_data) + value = lang_flat.get(key, '') + if value.startswith('[UNTRANSLATED]'): + value = '' + export_data['translations'][key][lang] = value + elif json_file.exists(): + lang_file = json_file lang_data = self._load_translation_file(lang_file) lang_flat = self._flatten_dict(lang_data) value = lang_flat.get(key, '') if value.startswith('[UNTRANSLATED]'): value = '' export_data['translations'][key][lang] = value - except SystemExit: - pass # Export files are always JSON with open(output_file, 'w', encoding='utf-8') as f: diff --git a/scripts/translations/compact_translator.py b/scripts/translations/compact_translator.py index 259bb2a86c..efe22f9f82 100644 --- a/scripts/translations/compact_translator.py +++ b/scripts/translations/compact_translator.py @@ -2,67 +2,37 @@ """ Compact Translation Extractor for Character-Limited AI Translation Outputs untranslated entries in minimal JSON format with whitespace stripped. -Supports both TOML and JSON formats. +TOML format only. """ import json import sys from pathlib import Path import argparse -try: - import tomllib # Python 3.11+ -except ImportError: - try: - import toml as tomllib_fallback - tomllib = None - except ImportError: - tomllib = None - tomllib_fallback = None +import tomllib # Python 3.11+ (stdlib) class CompactTranslationExtractor: def __init__(self, locales_dir: str = "frontend/public/locales", ignore_file: str = "scripts/ignore_translation.toml"): self.locales_dir = Path(locales_dir) - # Try TOML first, then fall back to JSON - self.golden_truth_file = self._find_translation_file(self.locales_dir / "en-GB") + self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" + if not self.golden_truth_file.exists(): + print(f"Error: en-GB translation file not found at {self.golden_truth_file}", file=sys.stderr) + sys.exit(1) self.golden_truth = self._load_translation_file(self.golden_truth_file) self.ignore_file = Path(ignore_file) self.ignore_patterns = self._load_ignore_patterns() - def _find_translation_file(self, lang_dir: Path) -> Path: - """Find translation file (TOML or JSON) in language directory.""" - toml_file = lang_dir / "translation.toml" - json_file = lang_dir / "translation.json" - - if toml_file.exists(): - return toml_file - elif json_file.exists(): - return json_file - else: - print(f"Error: No translation file found in {lang_dir}", file=sys.stderr) - sys.exit(1) - def _load_translation_file(self, file_path: Path) -> dict: - """Load TOML or JSON translation file based on extension.""" + """Load TOML translation file.""" try: - if file_path.suffix == '.toml': - if tomllib: - with open(file_path, 'rb') as f: - return tomllib.load(f) - elif tomllib_fallback: - with open(file_path, 'r', encoding='utf-8') as f: - return tomllib_fallback.load(f) - else: - print(f"Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+", file=sys.stderr) - sys.exit(1) - else: # JSON - with open(file_path, 'r', encoding='utf-8') as f: - return json.load(f) + with open(file_path, 'rb') as f: + return tomllib.load(f) except FileNotFoundError: print(f"Error: File not found: {file_path}", file=sys.stderr) sys.exit(1) except Exception as e: - print(f"Error: Invalid file {file_path}: {e}", file=sys.stderr) + print(f"Error: Invalid TOML file {file_path}: {e}", file=sys.stderr) sys.exit(1) def _load_ignore_patterns(self) -> dict: @@ -71,40 +41,13 @@ class CompactTranslationExtractor: return {} try: - if tomllib: - with open(self.ignore_file, 'rb') as f: - ignore_data = tomllib.load(f) - elif tomllib_fallback: - ignore_data = tomllib_fallback.load(self.ignore_file) - else: - ignore_data = self._parse_simple_toml() - + with open(self.ignore_file, 'rb') as f: + ignore_data = tomllib.load(f) return {lang: set(data.get('ignore', [])) for lang, data in ignore_data.items()} except Exception as e: print(f"Warning: Could not load ignore file {self.ignore_file}: {e}", file=sys.stderr) return {} - def _parse_simple_toml(self) -> dict: - """Simple TOML parser for ignore patterns (fallback).""" - ignore_data = {} - current_section = None - - with open(self.ignore_file, 'r', encoding='utf-8') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - - if line.startswith('[') and line.endswith(']'): - current_section = line[1:-1] - ignore_data[current_section] = {'ignore': []} - elif line.strip().startswith("'") and current_section: - item = line.strip().strip("',") - if item: - ignore_data[current_section]['ignore'].append(item) - - return ignore_data - def _flatten_dict(self, d: dict, parent_key: str = '', separator: str = '.') -> dict: """Flatten nested dictionary into dot-notation keys.""" items = [] @@ -119,9 +62,9 @@ class CompactTranslationExtractor: def get_untranslated_entries(self, language: str) -> dict: """Get all untranslated entries for a language in compact format.""" lang_dir = self.locales_dir / language - try: - target_file = self._find_translation_file(lang_dir) - except SystemExit: + target_file = lang_dir / "translation.toml" + + if not target_file.exists(): print(f"Error: Translation file not found for language: {language}", file=sys.stderr) sys.exit(1) @@ -173,8 +116,7 @@ class CompactTranslationExtractor: def main(): parser = argparse.ArgumentParser( - description='Extract untranslated entries in compact format for AI translation (supports TOML and JSON)', - epilog='Automatically detects and handles both TOML and JSON translation files.' + description='Extract untranslated entries in compact format for AI translation (TOML format only)' ) parser.add_argument('language', help='Language code (e.g., de-DE, fr-FR)') parser.add_argument('--locales-dir', default='frontend/public/locales', help='Path to locales directory') diff --git a/scripts/translations/validate_json_structure.py b/scripts/translations/validate_json_structure.py index 023230c4e2..102bc154c9 100644 --- a/scripts/translations/validate_json_structure.py +++ b/scripts/translations/validate_json_structure.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """ -Validate JSON/TOML structure and formatting of translation files. +Validate TOML structure and formatting of translation files. Checks for: -- Valid JSON/TOML syntax +- Valid TOML syntax - Consistent key structure with en-GB - Missing keys - Extra keys not in en-GB @@ -18,16 +18,7 @@ import sys from pathlib import Path from typing import Dict, List, Set import argparse - -try: - import tomllib # Python 3.11+ -except ImportError: - try: - import toml as tomllib_fallback - tomllib = None - except ImportError: - tomllib = None - tomllib_fallback = None +import tomllib # Python 3.11+ (stdlib) def get_all_keys(d: dict, parent_key: str = '', sep: str = '.') -> Set[str]: @@ -42,24 +33,11 @@ def get_all_keys(d: dict, parent_key: str = '', sep: str = '.') -> Set[str]: def validate_translation_file(file_path: Path) -> tuple[bool, str]: - """Validate that a file contains valid JSON or TOML.""" + """Validate that a file contains valid TOML.""" try: - if file_path.suffix == '.toml': - if tomllib: - with open(file_path, 'rb') as f: - tomllib.load(f) - elif tomllib_fallback: - with open(file_path, 'r', encoding='utf-8') as f: - tomllib_fallback.load(f) - else: - return False, "TOML support not available. Install 'toml' or upgrade to Python 3.11+" - return True, "Valid TOML" - else: # JSON - with open(file_path, 'r', encoding='utf-8') as f: - json.load(f) - return True, "Valid JSON" - except json.JSONDecodeError as e: - return False, f"Invalid JSON at line {e.lineno}, column {e.colno}: {e.msg}" + with open(file_path, 'rb') as f: + tomllib.load(f) + return True, "Valid TOML" except Exception as e: return False, f"Error reading file: {str(e)}" @@ -123,24 +101,14 @@ def print_validation_result(result: Dict, verbose: bool = False): def load_translation_file(file_path: Path) -> dict: - """Load JSON or TOML translation file.""" - if file_path.suffix == '.toml': - if tomllib: - with open(file_path, 'rb') as f: - return tomllib.load(f) - elif tomllib_fallback: - with open(file_path, 'r', encoding='utf-8') as f: - return tomllib_fallback.load(f) - else: - raise RuntimeError("TOML support not available") - else: # JSON - with open(file_path, 'r', encoding='utf-8') as f: - return json.load(f) + """Load TOML translation file.""" + with open(file_path, 'rb') as f: + return tomllib.load(f) def main(): parser = argparse.ArgumentParser( - description='Validate translation JSON/TOML structure' + description='Validate translation TOML structure' ) parser.add_argument( '--language', @@ -162,19 +130,11 @@ def main(): # Define paths locales_dir = Path('frontend/public/locales') + en_gb_path = locales_dir / 'en-GB' / 'translation.toml' + file_ext = '.toml' - # Try TOML first, then JSON - en_gb_toml = locales_dir / 'en-GB' / 'translation.toml' - en_gb_json = locales_dir / 'en-GB' / 'translation.json' - - if en_gb_toml.exists(): - en_gb_path = en_gb_toml - file_ext = '.toml' - elif en_gb_json.exists(): - en_gb_path = en_gb_json - file_ext = '.json' - else: - print(f"❌ Error: en-GB translation file not found at {en_gb_toml} or {en_gb_json}") + if not en_gb_path.exists(): + print(f"❌ Error: en-GB translation file not found at {en_gb_path}") sys.exit(1) # Validate en-GB itself @@ -196,7 +156,7 @@ def main(): languages = [] for d in locales_dir.iterdir(): if d.is_dir() and d.name != 'en-GB': - if (d / f'translation{file_ext}').exists(): + if (d / 'translation.toml').exists(): languages.append(d.name) results = [] @@ -204,10 +164,10 @@ def main(): # Validate each language for lang_code in sorted(languages): - lang_path = locales_dir / lang_code / f'translation{file_ext}' + lang_path = locales_dir / lang_code / 'translation.toml' if not lang_path.exists(): - print(f"⚠️ Warning: {lang_code}/translation{file_ext} not found, skipping") + print(f"⚠️ Warning: {lang_code}/translation.toml not found, skipping") continue # First check if file is valid diff --git a/scripts/translations/validate_placeholders.py b/scripts/translations/validate_placeholders.py index 15959ad466..5ce18d288f 100644 --- a/scripts/translations/validate_placeholders.py +++ b/scripts/translations/validate_placeholders.py @@ -15,16 +15,7 @@ import sys from pathlib import Path from typing import Dict, List, Set, Tuple import argparse - -try: - import tomllib # Python 3.11+ -except ImportError: - try: - import toml as tomllib_fallback - tomllib = None - except ImportError: - tomllib = None - tomllib_fallback = None +import tomllib # Python 3.11+ (stdlib) def find_placeholders(text: str) -> Set[str]: @@ -127,35 +118,16 @@ def main(): # Define paths locales_dir = Path('frontend/public/locales') + en_gb_path = locales_dir / 'en-GB' / 'translation.toml' + file_ext = '.toml' - # Try TOML first, then JSON - en_gb_toml = locales_dir / 'en-GB' / 'translation.toml' - en_gb_json = locales_dir / 'en-GB' / 'translation.json' - - if en_gb_toml.exists(): - en_gb_path = en_gb_toml - file_ext = '.toml' - elif en_gb_json.exists(): - en_gb_path = en_gb_json - file_ext = '.json' - else: - print(f"❌ Error: en-GB translation file not found at {en_gb_toml} or {en_gb_json}") + if not en_gb_path.exists(): + print(f"❌ Error: en-GB translation file not found at {en_gb_path}") sys.exit(1) # Load en-GB (source of truth) - if file_ext == '.toml': - if tomllib: - with open(en_gb_path, 'rb') as f: - en_gb = tomllib.load(f) - elif tomllib_fallback: - with open(en_gb_path, 'r', encoding='utf-8') as f: - en_gb = tomllib_fallback.load(f) - else: - print("❌ Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+") - sys.exit(1) - else: - with open(en_gb_path, 'r', encoding='utf-8') as f: - en_gb = json.load(f) + with open(en_gb_path, 'rb') as f: + en_gb = tomllib.load(f) en_gb_flat = flatten_dict(en_gb) @@ -167,33 +139,22 @@ def main(): languages = [] for d in locales_dir.iterdir(): if d.is_dir() and d.name != 'en-GB': - if (d / f'translation{file_ext}').exists(): + if (d / 'translation.toml').exists(): languages.append(d.name) all_issues = [] # Validate each language for lang_code in sorted(languages): - lang_path = locales_dir / lang_code / f'translation{file_ext}' + lang_path = locales_dir / lang_code / 'translation.toml' if not lang_path.exists(): - print(f"⚠️ Warning: {lang_code}/translation{file_ext} not found, skipping") + print(f"⚠️ Warning: {lang_code}/translation.toml not found, skipping") continue # Load language file - if file_ext == '.toml': - if tomllib: - with open(lang_path, 'rb') as f: - lang_data = tomllib.load(f) - elif tomllib_fallback: - with open(lang_path, 'r', encoding='utf-8') as f: - lang_data = tomllib_fallback.load(f) - else: - print(f"⚠️ Warning: Cannot read TOML file {lang_path}, skipping") - continue - else: - with open(lang_path, 'r', encoding='utf-8') as f: - lang_data = json.load(f) + with open(lang_path, 'rb') as f: + lang_data = tomllib.load(f) lang_flat = flatten_dict(lang_data) issues = validate_language(en_gb_flat, lang_flat, lang_code)