haeaeae/Stirling-PDF
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Anthony Stirling
2025-12-02 12:41:07 +00:00
parent 4829ed3dbe
commit 36b40c8d75
8 changed files with 118 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/check_language_toml.py
+8 -39
View File
@@ -20,21 +20,8 @@ import os
import argparse
import re
import json
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
try:
import tomli_w # For writing TOML files
except ImportError:
tomli_w = None
import tomllib # Python 3.11+ (stdlib)
import tomli_w # For writing TOML files
def find_duplicate_keys(file_path, keys=None, prefix=""):
@@ -51,14 +38,8 @@ def find_duplicate_keys(file_path, keys=None, prefix=""):
duplicates = []
# Load TOML file
if tomllib:
with open(file_path, 'rb') as file:
data = tomllib.load(file)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as file:
data = tomllib_fallback.load(file)
else:
return [] # Cannot check without TOML support
with open(file_path, 'rb') as file:
data = tomllib.load(file)
def process_dict(obj, current_prefix=""):
for key, value in obj.items():
@@ -86,14 +67,8 @@ def parse_toml_file(file_path):
:param file_path: Path to the TOML file.
:return: Dictionary with flattened keys.
"""
if tomllib:
with open(file_path, 'rb') as file:
data = tomllib.load(file)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as file:
data = tomllib_fallback.load(file)
else:
raise RuntimeError("TOML support not available. Install 'toml' or upgrade to Python 3.11+")
with open(file_path, 'rb') as file:
data = tomllib.load(file)
def flatten_dict(d, parent_key="", sep="."):
items = {}
@@ -135,14 +110,8 @@ def write_toml_file(file_path, updated_properties):
"""
nested_data = unflatten_dict(updated_properties)
if tomli_w:
with open(file_path, "wb") as file:
tomli_w.dump(nested_data, file)
elif tomllib_fallback and hasattr(tomllib_fallback, 'dump'):
with open(file_path, "w", encoding="utf-8", newline="\n") as file:
tomllib_fallback.dump(nested_data, file)
else:
raise RuntimeError("TOML writing not supported. Install 'tomli-w' library")
with open(file_path, "wb") as file:
tomli_w.dump(nested_data, file)
def update_missing_keys(reference_file, file_list, branch=""):
.github/workflows/check_toml.yml
+3 -6
View File
@@ -1,9 +1,6 @@
name: Check TOML Translation Files on PR (V3)
name: Check TOML Translation Files on PR
# This workflow validates TOML translation files for V3 format
# For legacy formats, see:
# - check_properties.yml (V1 backend .properties)
# - Legacy JSON validation (V2, deprecated)
# This workflow validates TOML translation files
on:
pull_request_target:
@@ -203,7 +200,7 @@ jobs:
- name: Install Python dependencies
run: |
pip install toml tomli-w
pip install tomli-w
- name: Run Python script to check files
id: run-check
.github/workflows/sync_files_v2.yml
+5 -5
View File
@@ -1,4 +1,4 @@
name: Sync Files V3 (TOML)
name: Sync Files (TOML)
on:
workflow_dispatch:
@@ -54,7 +54,7 @@ jobs:
- name: Install Python dependencies
run: |
pip install toml tomli-w
pip install tomli-w
- name: Sync translation TOML files
run: |
@@ -88,17 +88,17 @@ jobs:
signoff: true
branch: sync_readme_v3
base: main
title: ":globe_with_meridians: [V3] Sync Translations + Update README Progress Table"
title: ":globe_with_meridians: Sync Translations + Update README Progress Table"
body: |
### Description of Changes
This Pull Request was automatically generated to synchronize updates to translation files and documentation for **V3 (TOML format)**. Below are the details of the changes made:
This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made:
#### **1. Synchronization of Translation Files**
- Updated translation files (`frontend/public/locales/*/translation.toml`) to reflect changes in the reference file `en-GB/translation.toml`.
- Ensured consistency and synchronization across all supported language files.
- Highlighted any missing or incomplete translations.
- **Format**: TOML (V3, current) - migrated from JSON (V2) and .properties (V1)
- **Format**: TOML
#### **2. Update README.md**
- Generated the translation progress table in `README.md` using `counter_translation_v3.py`.
ADDING_TOOLS.md
+3 -3
View File
@@ -202,10 +202,10 @@ const [ToolName] = (props: BaseToolProps) => {
## 5. Add Translations
Update translation files. **Important: Only update `en-GB` files** - other languages are handled separately.
**File to update:** `frontend/public/locales/en-GB/translation.json`
**File to update:** `frontend/public/locales/en-GB/translation.toml`
**Required Translation Keys**:
```json
```toml
{
"home": {
"[toolName]": {
@@ -251,7 +251,7 @@ Update translation files. **Important: Only update `en-GB` files** - other langu
```
**Translation Notes:**
- **Only update `en-GB/translation.json`** - other locale files are managed separately
- **Only update `en-GB/translation.toml`** - other locale files are managed separately
- Use descriptive keys that match your component's `t()` calls
- Include tooltip translations if you created tooltip hooks
- Add `options.*` keys if your tool has settings with descriptions
scripts/translations/ai_translation_helper.py
+53 -43
View File
@@ -15,21 +15,8 @@ import argparse
import re
from datetime import datetime
import csv
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
try:
import tomli_w # For writing TOML
except ImportError:
tomli_w = None
import tomllib # Python 3.11+ (stdlib)
import tomli_w # For writing TOML
class AITranslationHelper:
@@ -55,15 +42,8 @@ class AITranslationHelper:
"""Load TOML or JSON translation file based on extension."""
try:
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
return tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
return tomllib_fallback.load(f)
else:
print(f"Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+")
sys.exit(1)
with open(file_path, 'rb') as f:
return tomllib.load(f)
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
@@ -74,12 +54,8 @@ class AITranslationHelper:
def _save_translation_file(self, data: Dict, file_path: Path) -> None:
"""Save translation file (TOML or JSON) based on extension."""
if file_path.suffix == '.toml':
if tomli_w:
with open(file_path, 'wb') as f:
tomli_w.dump(data, f)
else:
print(f"Error: TOML writing not available. Install 'tomli_w'")
sys.exit(1)
with open(file_path, 'wb') as f:
tomli_w.dump(data, f)
else: # JSON
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
@@ -106,10 +82,18 @@ class AITranslationHelper:
for lang in languages:
lang_dir = self.locales_dir / lang
try:
lang_file = self._find_translation_file(lang_dir)
# Check if translation file exists, if not create empty structure
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
except SystemExit:
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
else:
# No translation file found, create empty structure
lang_data = {}
@@ -285,10 +269,16 @@ class AITranslationHelper:
lang_dir = self.locales_dir / lang
# Load existing data or create new
try:
lang_file = self._find_translation_file(lang_dir)
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
except SystemExit:
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
else:
# No translation file found, create new JSON file
lang_data = {}
lang_dir.mkdir(parents=True, exist_ok=True)
@@ -347,15 +337,26 @@ class AITranslationHelper:
for lang in languages:
lang_dir = self.locales_dir / lang
try:
lang_file = self._find_translation_file(lang_dir)
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
row[lang] = value
except SystemExit:
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
row[lang] = value
else:
row[lang] = ''
writer.writerow(row)
@@ -377,16 +378,25 @@ class AITranslationHelper:
for lang in languages:
lang_dir = self.locales_dir / lang
try:
lang_file = self._find_translation_file(lang_dir)
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
export_data['translations'][key][lang] = value
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
export_data['translations'][key][lang] = value
except SystemExit:
pass
# Export files are always JSON
with open(output_file, 'w', encoding='utf-8') as f:
scripts/translations/compact_translator.py
+16 -74
View File
@@ -2,67 +2,37 @@
"""
Compact Translation Extractor for Character-Limited AI Translation
Outputs untranslated entries in minimal JSON format with whitespace stripped.
Supports both TOML and JSON formats.
TOML format only.
"""
import json
import sys
from pathlib import Path
import argparse
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
import tomllib # Python 3.11+ (stdlib)
class CompactTranslationExtractor:
def __init__(self, locales_dir: str = "frontend/public/locales", ignore_file: str = "scripts/ignore_translation.toml"):
self.locales_dir = Path(locales_dir)
# Try TOML first, then fall back to JSON
self.golden_truth_file = self._find_translation_file(self.locales_dir / "en-GB")
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml"
if not self.golden_truth_file.exists():
print(f"Error: en-GB translation file not found at {self.golden_truth_file}", file=sys.stderr)
sys.exit(1)
self.golden_truth = self._load_translation_file(self.golden_truth_file)
self.ignore_file = Path(ignore_file)
self.ignore_patterns = self._load_ignore_patterns()
def _find_translation_file(self, lang_dir: Path) -> Path:
"""Find translation file (TOML or JSON) in language directory."""
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
return toml_file
elif json_file.exists():
return json_file
else:
print(f"Error: No translation file found in {lang_dir}", file=sys.stderr)
sys.exit(1)
def _load_translation_file(self, file_path: Path) -> dict:
"""Load TOML or JSON translation file based on extension."""
"""Load TOML translation file."""
try:
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
return tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
return tomllib_fallback.load(f)
else:
print(f"Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+", file=sys.stderr)
sys.exit(1)
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
with open(file_path, 'rb') as f:
return tomllib.load(f)
except FileNotFoundError:
print(f"Error: File not found: {file_path}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"Error: Invalid file {file_path}: {e}", file=sys.stderr)
print(f"Error: Invalid TOML file {file_path}: {e}", file=sys.stderr)
sys.exit(1)
def _load_ignore_patterns(self) -> dict:
@@ -71,40 +41,13 @@ class CompactTranslationExtractor:
return {}
try:
if tomllib:
with open(self.ignore_file, 'rb') as f:
ignore_data = tomllib.load(f)
elif tomllib_fallback:
ignore_data = tomllib_fallback.load(self.ignore_file)
else:
ignore_data = self._parse_simple_toml()
with open(self.ignore_file, 'rb') as f:
ignore_data = tomllib.load(f)
return {lang: set(data.get('ignore', [])) for lang, data in ignore_data.items()}
except Exception as e:
print(f"Warning: Could not load ignore file {self.ignore_file}: {e}", file=sys.stderr)
return {}
def _parse_simple_toml(self) -> dict:
"""Simple TOML parser for ignore patterns (fallback)."""
ignore_data = {}
current_section = None
with open(self.ignore_file, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
if line.startswith('[') and line.endswith(']'):
current_section = line[1:-1]
ignore_data[current_section] = {'ignore': []}
elif line.strip().startswith("'") and current_section:
item = line.strip().strip("',")
if item:
ignore_data[current_section]['ignore'].append(item)
return ignore_data
def _flatten_dict(self, d: dict, parent_key: str = '', separator: str = '.') -> dict:
"""Flatten nested dictionary into dot-notation keys."""
items = []
@@ -119,9 +62,9 @@ class CompactTranslationExtractor:
def get_untranslated_entries(self, language: str) -> dict:
"""Get all untranslated entries for a language in compact format."""
lang_dir = self.locales_dir / language
try:
target_file = self._find_translation_file(lang_dir)
except SystemExit:
target_file = lang_dir / "translation.toml"
if not target_file.exists():
print(f"Error: Translation file not found for language: {language}", file=sys.stderr)
sys.exit(1)
@@ -173,8 +116,7 @@ class CompactTranslationExtractor:
def main():
parser = argparse.ArgumentParser(
description='Extract untranslated entries in compact format for AI translation (supports TOML and JSON)',
epilog='Automatically detects and handles both TOML and JSON translation files.'
description='Extract untranslated entries in compact format for AI translation (TOML format only)'
)
parser.add_argument('language', help='Language code (e.g., de-DE, fr-FR)')
parser.add_argument('--locales-dir', default='frontend/public/locales', help='Path to locales directory')
scripts/translations/validate_json_structure.py
+18 -58
View File
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
"""
Validate JSON/TOML structure and formatting of translation files.
Validate TOML structure and formatting of translation files.
Checks for:
- Valid JSON/TOML syntax
- Valid TOML syntax
- Consistent key structure with en-GB
- Missing keys
- Extra keys not in en-GB
@@ -18,16 +18,7 @@ import sys
from pathlib import Path
from typing import Dict, List, Set
import argparse
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
import tomllib # Python 3.11+ (stdlib)
def get_all_keys(d: dict, parent_key: str = '', sep: str = '.') -> Set[str]:
@@ -42,24 +33,11 @@ def get_all_keys(d: dict, parent_key: str = '', sep: str = '.') -> Set[str]:
def validate_translation_file(file_path: Path) -> tuple[bool, str]:
"""Validate that a file contains valid JSON or TOML."""
"""Validate that a file contains valid TOML."""
try:
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
tomllib_fallback.load(f)
else:
return False, "TOML support not available. Install 'toml' or upgrade to Python 3.11+"
return True, "Valid TOML"
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
json.load(f)
return True, "Valid JSON"
except json.JSONDecodeError as e:
return False, f"Invalid JSON at line {e.lineno}, column {e.colno}: {e.msg}"
with open(file_path, 'rb') as f:
tomllib.load(f)
return True, "Valid TOML"
except Exception as e:
return False, f"Error reading file: {str(e)}"
@@ -123,24 +101,14 @@ def print_validation_result(result: Dict, verbose: bool = False):
def load_translation_file(file_path: Path) -> dict:
"""Load JSON or TOML translation file."""
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
return tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
return tomllib_fallback.load(f)
else:
raise RuntimeError("TOML support not available")
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
"""Load TOML translation file."""
with open(file_path, 'rb') as f:
return tomllib.load(f)
def main():
parser = argparse.ArgumentParser(
description='Validate translation JSON/TOML structure'
description='Validate translation TOML structure'
)
parser.add_argument(
'--language',
@@ -162,19 +130,11 @@ def main():
# Define paths
locales_dir = Path('frontend/public/locales')
en_gb_path = locales_dir / 'en-GB' / 'translation.toml'
file_ext = '.toml'
# Try TOML first, then JSON
en_gb_toml = locales_dir / 'en-GB' / 'translation.toml'
en_gb_json = locales_dir / 'en-GB' / 'translation.json'
if en_gb_toml.exists():
en_gb_path = en_gb_toml
file_ext = '.toml'
elif en_gb_json.exists():
en_gb_path = en_gb_json
file_ext = '.json'
else:
print(f"❌ Error: en-GB translation file not found at {en_gb_toml} or {en_gb_json}")
if not en_gb_path.exists():
print(f"❌ Error: en-GB translation file not found at {en_gb_path}")
sys.exit(1)
# Validate en-GB itself
@@ -196,7 +156,7 @@ def main():
languages = []
for d in locales_dir.iterdir():
if d.is_dir() and d.name != 'en-GB':
if (d / f'translation{file_ext}').exists():
if (d / 'translation.toml').exists():
languages.append(d.name)
results = []
@@ -204,10 +164,10 @@ def main():
# Validate each language
for lang_code in sorted(languages):
lang_path = locales_dir / lang_code / f'translation{file_ext}'
lang_path = locales_dir / lang_code / 'translation.toml'
if not lang_path.exists():
print(f"⚠️ Warning: {lang_code}/translation{file_ext} not found, skipping")
print(f"⚠️ Warning: {lang_code}/translation.toml not found, skipping")
continue
# First check if file is valid
scripts/translations/validate_placeholders.py
+12 -51
View File
@@ -15,16 +15,7 @@ import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
import argparse
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
import tomllib # Python 3.11+ (stdlib)
def find_placeholders(text: str) -> Set[str]:
@@ -127,35 +118,16 @@ def main():
# Define paths
locales_dir = Path('frontend/public/locales')
en_gb_path = locales_dir / 'en-GB' / 'translation.toml'
file_ext = '.toml'
# Try TOML first, then JSON
en_gb_toml = locales_dir / 'en-GB' / 'translation.toml'
en_gb_json = locales_dir / 'en-GB' / 'translation.json'
if en_gb_toml.exists():
en_gb_path = en_gb_toml
file_ext = '.toml'
elif en_gb_json.exists():
en_gb_path = en_gb_json
file_ext = '.json'
else:
print(f"❌ Error: en-GB translation file not found at {en_gb_toml} or {en_gb_json}")
if not en_gb_path.exists():
print(f"❌ Error: en-GB translation file not found at {en_gb_path}")
sys.exit(1)
# Load en-GB (source of truth)
if file_ext == '.toml':
if tomllib:
with open(en_gb_path, 'rb') as f:
en_gb = tomllib.load(f)
elif tomllib_fallback:
with open(en_gb_path, 'r', encoding='utf-8') as f:
en_gb = tomllib_fallback.load(f)
else:
print("❌ Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+")
sys.exit(1)
else:
with open(en_gb_path, 'r', encoding='utf-8') as f:
en_gb = json.load(f)
with open(en_gb_path, 'rb') as f:
en_gb = tomllib.load(f)
en_gb_flat = flatten_dict(en_gb)
@@ -167,33 +139,22 @@ def main():
languages = []
for d in locales_dir.iterdir():
if d.is_dir() and d.name != 'en-GB':
if (d / f'translation{file_ext}').exists():
if (d / 'translation.toml').exists():
languages.append(d.name)
all_issues = []
# Validate each language
for lang_code in sorted(languages):
lang_path = locales_dir / lang_code / f'translation{file_ext}'
lang_path = locales_dir / lang_code / 'translation.toml'
if not lang_path.exists():
print(f"⚠️ Warning: {lang_code}/translation{file_ext} not found, skipping")
print(f"⚠️ Warning: {lang_code}/translation.toml not found, skipping")
continue
# Load language file
if file_ext == '.toml':
if tomllib:
with open(lang_path, 'rb') as f:
lang_data = tomllib.load(f)
elif tomllib_fallback:
with open(lang_path, 'r', encoding='utf-8') as f:
lang_data = tomllib_fallback.load(f)
else:
print(f"⚠️ Warning: Cannot read TOML file {lang_path}, skipping")
continue
else:
with open(lang_path, 'r', encoding='utf-8') as f:
lang_data = json.load(f)
with open(lang_path, 'rb') as f:
lang_data = tomllib.load(f)
lang_flat = flatten_dict(lang_data)
issues = validate_language(en_gb_flat, lang_flat, lang_code)