haeaeae/Stirling-PDF
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Anthony Stirling
2025-12-02 12:41:07 +00:00
parent 4829ed3dbe
commit 36b40c8d75
8 changed files with 118 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/translations/ai_translation_helper.py
+53 -43
View File
@@ -15,21 +15,8 @@ import argparse
import re
from datetime import datetime
import csv
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
try:
import tomli_w # For writing TOML
except ImportError:
tomli_w = None
import tomllib # Python 3.11+ (stdlib)
import tomli_w # For writing TOML
class AITranslationHelper:
@@ -55,15 +42,8 @@ class AITranslationHelper:
"""Load TOML or JSON translation file based on extension."""
try:
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
return tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
return tomllib_fallback.load(f)
else:
print(f"Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+")
sys.exit(1)
with open(file_path, 'rb') as f:
return tomllib.load(f)
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
@@ -74,12 +54,8 @@ class AITranslationHelper:
def _save_translation_file(self, data: Dict, file_path: Path) -> None:
"""Save translation file (TOML or JSON) based on extension."""
if file_path.suffix == '.toml':
if tomli_w:
with open(file_path, 'wb') as f:
tomli_w.dump(data, f)
else:
print(f"Error: TOML writing not available. Install 'tomli_w'")
sys.exit(1)
with open(file_path, 'wb') as f:
tomli_w.dump(data, f)
else: # JSON
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
@@ -106,10 +82,18 @@ class AITranslationHelper:
for lang in languages:
lang_dir = self.locales_dir / lang
try:
lang_file = self._find_translation_file(lang_dir)
# Check if translation file exists, if not create empty structure
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
except SystemExit:
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
else:
# No translation file found, create empty structure
lang_data = {}
@@ -285,10 +269,16 @@ class AITranslationHelper:
lang_dir = self.locales_dir / lang
# Load existing data or create new
try:
lang_file = self._find_translation_file(lang_dir)
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
except SystemExit:
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
else:
# No translation file found, create new JSON file
lang_data = {}
lang_dir.mkdir(parents=True, exist_ok=True)
@@ -347,15 +337,26 @@ class AITranslationHelper:
for lang in languages:
lang_dir = self.locales_dir / lang
try:
lang_file = self._find_translation_file(lang_dir)
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
row[lang] = value
except SystemExit:
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
row[lang] = value
else:
row[lang] = ''
writer.writerow(row)
@@ -377,16 +378,25 @@ class AITranslationHelper:
for lang in languages:
lang_dir = self.locales_dir / lang
try:
lang_file = self._find_translation_file(lang_dir)
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
lang_file = toml_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
export_data['translations'][key][lang] = value
elif json_file.exists():
lang_file = json_file
lang_data = self._load_translation_file(lang_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
export_data['translations'][key][lang] = value
except SystemExit:
pass
# Export files are always JSON
with open(output_file, 'w', encoding='utf-8') as f:
scripts/translations/compact_translator.py
+16 -74
View File
@@ -2,67 +2,37 @@
"""
Compact Translation Extractor for Character-Limited AI Translation
Outputs untranslated entries in minimal JSON format with whitespace stripped.
Supports both TOML and JSON formats.
TOML format only.
"""
import json
import sys
from pathlib import Path
import argparse
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
import tomllib # Python 3.11+ (stdlib)
class CompactTranslationExtractor:
def __init__(self, locales_dir: str = "frontend/public/locales", ignore_file: str = "scripts/ignore_translation.toml"):
self.locales_dir = Path(locales_dir)
# Try TOML first, then fall back to JSON
self.golden_truth_file = self._find_translation_file(self.locales_dir / "en-GB")
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml"
if not self.golden_truth_file.exists():
print(f"Error: en-GB translation file not found at {self.golden_truth_file}", file=sys.stderr)
sys.exit(1)
self.golden_truth = self._load_translation_file(self.golden_truth_file)
self.ignore_file = Path(ignore_file)
self.ignore_patterns = self._load_ignore_patterns()
def _find_translation_file(self, lang_dir: Path) -> Path:
"""Find translation file (TOML or JSON) in language directory."""
toml_file = lang_dir / "translation.toml"
json_file = lang_dir / "translation.json"
if toml_file.exists():
return toml_file
elif json_file.exists():
return json_file
else:
print(f"Error: No translation file found in {lang_dir}", file=sys.stderr)
sys.exit(1)
def _load_translation_file(self, file_path: Path) -> dict:
"""Load TOML or JSON translation file based on extension."""
"""Load TOML translation file."""
try:
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
return tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
return tomllib_fallback.load(f)
else:
print(f"Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+", file=sys.stderr)
sys.exit(1)
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
with open(file_path, 'rb') as f:
return tomllib.load(f)
except FileNotFoundError:
print(f"Error: File not found: {file_path}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"Error: Invalid file {file_path}: {e}", file=sys.stderr)
print(f"Error: Invalid TOML file {file_path}: {e}", file=sys.stderr)
sys.exit(1)
def _load_ignore_patterns(self) -> dict:
@@ -71,40 +41,13 @@ class CompactTranslationExtractor:
return {}
try:
if tomllib:
with open(self.ignore_file, 'rb') as f:
ignore_data = tomllib.load(f)
elif tomllib_fallback:
ignore_data = tomllib_fallback.load(self.ignore_file)
else:
ignore_data = self._parse_simple_toml()
with open(self.ignore_file, 'rb') as f:
ignore_data = tomllib.load(f)
return {lang: set(data.get('ignore', [])) for lang, data in ignore_data.items()}
except Exception as e:
print(f"Warning: Could not load ignore file {self.ignore_file}: {e}", file=sys.stderr)
return {}
def _parse_simple_toml(self) -> dict:
"""Simple TOML parser for ignore patterns (fallback)."""
ignore_data = {}
current_section = None
with open(self.ignore_file, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
if line.startswith('[') and line.endswith(']'):
current_section = line[1:-1]
ignore_data[current_section] = {'ignore': []}
elif line.strip().startswith("'") and current_section:
item = line.strip().strip("',")
if item:
ignore_data[current_section]['ignore'].append(item)
return ignore_data
def _flatten_dict(self, d: dict, parent_key: str = '', separator: str = '.') -> dict:
"""Flatten nested dictionary into dot-notation keys."""
items = []
@@ -119,9 +62,9 @@ class CompactTranslationExtractor:
def get_untranslated_entries(self, language: str) -> dict:
"""Get all untranslated entries for a language in compact format."""
lang_dir = self.locales_dir / language
try:
target_file = self._find_translation_file(lang_dir)
except SystemExit:
target_file = lang_dir / "translation.toml"
if not target_file.exists():
print(f"Error: Translation file not found for language: {language}", file=sys.stderr)
sys.exit(1)
@@ -173,8 +116,7 @@ class CompactTranslationExtractor:
def main():
parser = argparse.ArgumentParser(
description='Extract untranslated entries in compact format for AI translation (supports TOML and JSON)',
epilog='Automatically detects and handles both TOML and JSON translation files.'
description='Extract untranslated entries in compact format for AI translation (TOML format only)'
)
parser.add_argument('language', help='Language code (e.g., de-DE, fr-FR)')
parser.add_argument('--locales-dir', default='frontend/public/locales', help='Path to locales directory')
scripts/translations/validate_json_structure.py
+18 -58
View File
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
"""
Validate JSON/TOML structure and formatting of translation files.
Validate TOML structure and formatting of translation files.
Checks for:
- Valid JSON/TOML syntax
- Valid TOML syntax
- Consistent key structure with en-GB
- Missing keys
- Extra keys not in en-GB
@@ -18,16 +18,7 @@ import sys
from pathlib import Path
from typing import Dict, List, Set
import argparse
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
import tomllib # Python 3.11+ (stdlib)
def get_all_keys(d: dict, parent_key: str = '', sep: str = '.') -> Set[str]:
@@ -42,24 +33,11 @@ def get_all_keys(d: dict, parent_key: str = '', sep: str = '.') -> Set[str]:
def validate_translation_file(file_path: Path) -> tuple[bool, str]:
"""Validate that a file contains valid JSON or TOML."""
"""Validate that a file contains valid TOML."""
try:
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
tomllib_fallback.load(f)
else:
return False, "TOML support not available. Install 'toml' or upgrade to Python 3.11+"
return True, "Valid TOML"
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
json.load(f)
return True, "Valid JSON"
except json.JSONDecodeError as e:
return False, f"Invalid JSON at line {e.lineno}, column {e.colno}: {e.msg}"
with open(file_path, 'rb') as f:
tomllib.load(f)
return True, "Valid TOML"
except Exception as e:
return False, f"Error reading file: {str(e)}"
@@ -123,24 +101,14 @@ def print_validation_result(result: Dict, verbose: bool = False):
def load_translation_file(file_path: Path) -> dict:
"""Load JSON or TOML translation file."""
if file_path.suffix == '.toml':
if tomllib:
with open(file_path, 'rb') as f:
return tomllib.load(f)
elif tomllib_fallback:
with open(file_path, 'r', encoding='utf-8') as f:
return tomllib_fallback.load(f)
else:
raise RuntimeError("TOML support not available")
else: # JSON
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
"""Load TOML translation file."""
with open(file_path, 'rb') as f:
return tomllib.load(f)
def main():
parser = argparse.ArgumentParser(
description='Validate translation JSON/TOML structure'
description='Validate translation TOML structure'
)
parser.add_argument(
'--language',
@@ -162,19 +130,11 @@ def main():
# Define paths
locales_dir = Path('frontend/public/locales')
en_gb_path = locales_dir / 'en-GB' / 'translation.toml'
file_ext = '.toml'
# Try TOML first, then JSON
en_gb_toml = locales_dir / 'en-GB' / 'translation.toml'
en_gb_json = locales_dir / 'en-GB' / 'translation.json'
if en_gb_toml.exists():
en_gb_path = en_gb_toml
file_ext = '.toml'
elif en_gb_json.exists():
en_gb_path = en_gb_json
file_ext = '.json'
else:
print(f"❌ Error: en-GB translation file not found at {en_gb_toml} or {en_gb_json}")
if not en_gb_path.exists():
print(f"❌ Error: en-GB translation file not found at {en_gb_path}")
sys.exit(1)
# Validate en-GB itself
@@ -196,7 +156,7 @@ def main():
languages = []
for d in locales_dir.iterdir():
if d.is_dir() and d.name != 'en-GB':
if (d / f'translation{file_ext}').exists():
if (d / 'translation.toml').exists():
languages.append(d.name)
results = []
@@ -204,10 +164,10 @@ def main():
# Validate each language
for lang_code in sorted(languages):
lang_path = locales_dir / lang_code / f'translation{file_ext}'
lang_path = locales_dir / lang_code / 'translation.toml'
if not lang_path.exists():
print(f"⚠️ Warning: {lang_code}/translation{file_ext} not found, skipping")
print(f"⚠️ Warning: {lang_code}/translation.toml not found, skipping")
continue
# First check if file is valid
scripts/translations/validate_placeholders.py
+12 -51
View File
@@ -15,16 +15,7 @@ import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
import argparse
try:
import tomllib # Python 3.11+
except ImportError:
try:
import toml as tomllib_fallback
tomllib = None
except ImportError:
tomllib = None
tomllib_fallback = None
import tomllib # Python 3.11+ (stdlib)
def find_placeholders(text: str) -> Set[str]:
@@ -127,35 +118,16 @@ def main():
# Define paths
locales_dir = Path('frontend/public/locales')
en_gb_path = locales_dir / 'en-GB' / 'translation.toml'
file_ext = '.toml'
# Try TOML first, then JSON
en_gb_toml = locales_dir / 'en-GB' / 'translation.toml'
en_gb_json = locales_dir / 'en-GB' / 'translation.json'
if en_gb_toml.exists():
en_gb_path = en_gb_toml
file_ext = '.toml'
elif en_gb_json.exists():
en_gb_path = en_gb_json
file_ext = '.json'
else:
print(f"❌ Error: en-GB translation file not found at {en_gb_toml} or {en_gb_json}")
if not en_gb_path.exists():
print(f"❌ Error: en-GB translation file not found at {en_gb_path}")
sys.exit(1)
# Load en-GB (source of truth)
if file_ext == '.toml':
if tomllib:
with open(en_gb_path, 'rb') as f:
en_gb = tomllib.load(f)
elif tomllib_fallback:
with open(en_gb_path, 'r', encoding='utf-8') as f:
en_gb = tomllib_fallback.load(f)
else:
print("❌ Error: TOML support not available. Install 'toml' or upgrade to Python 3.11+")
sys.exit(1)
else:
with open(en_gb_path, 'r', encoding='utf-8') as f:
en_gb = json.load(f)
with open(en_gb_path, 'rb') as f:
en_gb = tomllib.load(f)
en_gb_flat = flatten_dict(en_gb)
@@ -167,33 +139,22 @@ def main():
languages = []
for d in locales_dir.iterdir():
if d.is_dir() and d.name != 'en-GB':
if (d / f'translation{file_ext}').exists():
if (d / 'translation.toml').exists():
languages.append(d.name)
all_issues = []
# Validate each language
for lang_code in sorted(languages):
lang_path = locales_dir / lang_code / f'translation{file_ext}'
lang_path = locales_dir / lang_code / 'translation.toml'
if not lang_path.exists():
print(f"⚠️ Warning: {lang_code}/translation{file_ext} not found, skipping")
print(f"⚠️ Warning: {lang_code}/translation.toml not found, skipping")
continue
# Load language file
if file_ext == '.toml':
if tomllib:
with open(lang_path, 'rb') as f:
lang_data = tomllib.load(f)
elif tomllib_fallback:
with open(lang_path, 'r', encoding='utf-8') as f:
lang_data = tomllib_fallback.load(f)
else:
print(f"⚠️ Warning: Cannot read TOML file {lang_path}, skipping")
continue
else:
with open(lang_path, 'r', encoding='utf-8') as f:
lang_data = json.load(f)
with open(lang_path, 'rb') as f:
lang_data = tomllib.load(f)
lang_flat = flatten_dict(lang_data)
issues = validate_language(en_gb_flat, lang_flat, lang_code)