edde746
f622ba8efe
scripts/ had ~80 flat files. Entry points (ci_*.sh, codegen.sh, run_tests.sh, format_native.sh, setup_hooks.sh, upload-symbols.*) and the shared pubspec_version.py stay at the root; checkers, generators, maestro tooling and release tooling move into subdirectories with their tests. Updated every reference: workflow steps, guard-test glob, Docker COPY paths and .dockerignore whitelist, website audit path, dart test imports, and regenerated the five outputs whose headers embed generator paths.
505 lines
18 KiB
Dart
505 lines
18 KiB
Dart
/// Generates lib/data/ducet_order.dart from pinned Unicode and CLDR data.
|
|
///
|
|
/// Usage:
|
|
/// dart run scripts/codegen/generate_ducet_ranks.dart [allkeys.txt] [FractionalUCA.txt]
|
|
///
|
|
/// Default generation uses tracked deterministic gzip copies of the pinned source bytes.
|
|
/// Unicode data is redistributed unmodified under the Unicode License v3.
|
|
/// Explicit/cache/download inputs remain available for integrity and HTTP failure testing.
|
|
library;
|
|
|
|
import 'dart:async';
|
|
import 'dart:io';
|
|
|
|
import 'package:crypto/crypto.dart';
|
|
|
|
final class SourceDescriptor {
|
|
const SourceDescriptor({
|
|
required this.name,
|
|
required this.url,
|
|
required this.bundledFileName,
|
|
required this.cacheFileName,
|
|
required this.sha256Digest,
|
|
required this.licenseUrl,
|
|
});
|
|
|
|
final String name;
|
|
final String url;
|
|
final String bundledFileName;
|
|
final String cacheFileName;
|
|
final String sha256Digest;
|
|
final String licenseUrl;
|
|
}
|
|
|
|
const allKeysSource = SourceDescriptor(
|
|
name: 'Unicode 13.0 allkeys',
|
|
url: 'https://www.unicode.org/Public/UCA/13.0.0/allkeys.txt',
|
|
bundledFileName: 'allkeys-13.0.0.txt.gz',
|
|
cacheFileName: 'plezy-allkeys-13.0.0-a3255d45b7af97f4dc14fb8364d7573b434425e5c58cacf00d16901ce081c78d.txt',
|
|
sha256Digest: 'a3255d45b7af97f4dc14fb8364d7573b434425e5c58cacf00d16901ce081c78d',
|
|
licenseUrl: 'https://www.unicode.org/license.txt',
|
|
);
|
|
|
|
const fractionalUcaSource = SourceDescriptor(
|
|
name: 'CLDR FractionalUCA at 651afecf9ccf1541a49306993e8210fa2209aa0b',
|
|
url:
|
|
'https://raw.githubusercontent.com/unicode-org/cldr/651afecf9ccf1541a49306993e8210fa2209aa0b/common/uca/FractionalUCA.txt',
|
|
bundledFileName: 'FractionalUCA-651afecf9ccf1541a49306993e8210fa2209aa0b.txt.gz',
|
|
cacheFileName:
|
|
'plezy-FractionalUCA-651afecf9ccf1541a49306993e8210fa2209aa0b-'
|
|
'a6144d0c8c19cc899a5d2f48fbc14e3e31e819049a73aa86b67111f1f3f81637.txt',
|
|
sha256Digest: 'a6144d0c8c19cc899a5d2f48fbc14e3e31e819049a73aa86b67111f1f3f81637',
|
|
licenseUrl: 'https://github.com/unicode-org/cldr/blob/651afecf9ccf1541a49306993e8210fa2209aa0b/LICENSE',
|
|
);
|
|
|
|
final class SourceDownloadResponse {
|
|
const SourceDownloadResponse({required this.statusCode, required this.bytes, this.close});
|
|
|
|
final int statusCode;
|
|
final Stream<List<int>> bytes;
|
|
final void Function()? close;
|
|
}
|
|
|
|
typedef SourceDownloader = Future<SourceDownloadResponse> Function(Uri uri);
|
|
typedef GeneratedFileWriter = void Function(File output, String contents);
|
|
|
|
final class SourceIntegrityException implements Exception {
|
|
const SourceIntegrityException(this.message);
|
|
|
|
final String message;
|
|
|
|
@override
|
|
String toString() => 'SourceIntegrityException: $message';
|
|
}
|
|
|
|
final class VerifiedSourceBundle {
|
|
VerifiedSourceBundle({
|
|
required this.allKeysFile,
|
|
required this.fractionalUcaFile,
|
|
required this._stagingDirectory,
|
|
required this._stagedCacheFiles,
|
|
});
|
|
|
|
final File allKeysFile;
|
|
final File fractionalUcaFile;
|
|
final Directory _stagingDirectory;
|
|
final Map<File, File> _stagedCacheFiles;
|
|
|
|
void commitCaches() {
|
|
for (final entry in _stagedCacheFiles.entries) {
|
|
final staged = entry.key;
|
|
final cache = entry.value;
|
|
cache.parent.createSync(recursive: true);
|
|
staged.renameSync(cache.path);
|
|
}
|
|
_stagedCacheFiles.clear();
|
|
}
|
|
|
|
void dispose() {
|
|
if (_stagingDirectory.existsSync()) {
|
|
_stagingDirectory.deleteSync(recursive: true);
|
|
}
|
|
}
|
|
}
|
|
|
|
Future<SourceDownloadResponse> downloadSource(Uri uri) async {
|
|
final client = HttpClient();
|
|
try {
|
|
final request = await client.getUrl(uri);
|
|
request.headers.set(HttpHeaders.userAgentHeader, 'Plezy DUCET generator');
|
|
final response = await request.close();
|
|
return SourceDownloadResponse(
|
|
statusCode: response.statusCode,
|
|
bytes: response,
|
|
close: () => client.close(force: true),
|
|
);
|
|
} catch (_) {
|
|
client.close(force: true);
|
|
rethrow;
|
|
}
|
|
}
|
|
|
|
Future<String> sha256ForFile(File file) async => (await sha256.bind(file.openRead()).first).toString();
|
|
|
|
Future<void> verifySourceFile(File file, SourceDescriptor descriptor) async {
|
|
if (!await file.exists()) {
|
|
throw SourceIntegrityException('${descriptor.name} is missing: ${file.path}');
|
|
}
|
|
final actualDigest = await sha256ForFile(file);
|
|
if (actualDigest != descriptor.sha256Digest) {
|
|
throw SourceIntegrityException(
|
|
'${descriptor.name} SHA-256 mismatch: expected ${descriptor.sha256Digest}, got $actualDigest',
|
|
);
|
|
}
|
|
}
|
|
|
|
Future<VerifiedSourceBundle> loadBundledSources({
|
|
required Directory bundledSourceDirectory,
|
|
SourceDescriptor allKeysDescriptor = allKeysSource,
|
|
SourceDescriptor fractionalUcaDescriptor = fractionalUcaSource,
|
|
}) async {
|
|
final staging = await Directory.systemTemp.createTemp('plezy_ducet_bundled_');
|
|
|
|
Future<File> decompressAndVerify(SourceDescriptor descriptor) async {
|
|
final compressed = File.fromUri(bundledSourceDirectory.uri.resolve(descriptor.bundledFileName));
|
|
if (!await compressed.exists()) {
|
|
throw SourceIntegrityException('${descriptor.name} bundled source is missing: ${compressed.path}');
|
|
}
|
|
final decompressed = File.fromUri(staging.uri.resolve('${descriptor.bundledFileName}.raw'));
|
|
try {
|
|
await compressed.openRead().transform(gzip.decoder).pipe(decompressed.openWrite());
|
|
} catch (error) {
|
|
throw SourceIntegrityException('${descriptor.name} bundled gzip is invalid: $error');
|
|
}
|
|
await verifySourceFile(decompressed, descriptor);
|
|
return decompressed;
|
|
}
|
|
|
|
try {
|
|
final allKeys = await decompressAndVerify(allKeysDescriptor);
|
|
final fractionalUca = await decompressAndVerify(fractionalUcaDescriptor);
|
|
return VerifiedSourceBundle(
|
|
allKeysFile: allKeys,
|
|
fractionalUcaFile: fractionalUca,
|
|
stagingDirectory: staging,
|
|
stagedCacheFiles: <File, File>{},
|
|
);
|
|
} catch (_) {
|
|
if (await staging.exists()) {
|
|
await staging.delete(recursive: true);
|
|
}
|
|
rethrow;
|
|
}
|
|
}
|
|
|
|
Future<VerifiedSourceBundle> loadVerifiedSources({
|
|
File? explicitAllKeys,
|
|
File? explicitFractionalUca,
|
|
Directory? cacheDirectory,
|
|
SourceDownloader downloader = downloadSource,
|
|
SourceDescriptor allKeysDescriptor = allKeysSource,
|
|
SourceDescriptor fractionalUcaDescriptor = fractionalUcaSource,
|
|
}) async {
|
|
final cacheRoot = cacheDirectory ?? Directory.systemTemp;
|
|
final staging = await Directory.systemTemp.createTemp('plezy_ducet_sources_');
|
|
final stagedCacheFiles = <File, File>{};
|
|
|
|
Future<File> resolve(SourceDescriptor descriptor, File? explicit) async {
|
|
if (explicit != null) {
|
|
await verifySourceFile(explicit, descriptor);
|
|
return explicit;
|
|
}
|
|
|
|
final cache = File.fromUri(cacheRoot.uri.resolve(descriptor.cacheFileName));
|
|
if (await cache.exists()) {
|
|
await verifySourceFile(cache, descriptor);
|
|
return cache;
|
|
}
|
|
|
|
final staged = File.fromUri(staging.uri.resolve(descriptor.cacheFileName));
|
|
final response = await downloader(Uri.parse(descriptor.url));
|
|
try {
|
|
if (response.statusCode != HttpStatus.ok) {
|
|
throw SourceIntegrityException('${descriptor.name} download returned HTTP ${response.statusCode}');
|
|
}
|
|
await response.bytes.pipe(staged.openWrite());
|
|
} finally {
|
|
response.close?.call();
|
|
}
|
|
await verifySourceFile(staged, descriptor);
|
|
stagedCacheFiles[staged] = cache;
|
|
return staged;
|
|
}
|
|
|
|
try {
|
|
final allKeys = await resolve(allKeysDescriptor, explicitAllKeys);
|
|
final fractionalUca = await resolve(fractionalUcaDescriptor, explicitFractionalUca);
|
|
return VerifiedSourceBundle(
|
|
allKeysFile: allKeys,
|
|
fractionalUcaFile: fractionalUca,
|
|
stagingDirectory: staging,
|
|
stagedCacheFiles: stagedCacheFiles,
|
|
);
|
|
} catch (_) {
|
|
if (await staging.exists()) {
|
|
await staging.delete(recursive: true);
|
|
}
|
|
rethrow;
|
|
}
|
|
}
|
|
|
|
class Weight implements Comparable<Weight> {
|
|
final List<(int, int, int)> levels;
|
|
final int codepoint;
|
|
|
|
const Weight(this.levels, this.codepoint);
|
|
|
|
@override
|
|
int compareTo(Weight other) {
|
|
final len = levels.length < other.levels.length ? levels.length : other.levels.length;
|
|
for (var i = 0; i < len; i++) {
|
|
final cmp = levels[i].$1.compareTo(other.levels[i].$1);
|
|
if (cmp != 0) return cmp;
|
|
}
|
|
if (levels.length != other.levels.length) {
|
|
return levels.length.compareTo(other.levels.length);
|
|
}
|
|
for (var i = 0; i < len; i++) {
|
|
final cmp = levels[i].$2.compareTo(other.levels[i].$2);
|
|
if (cmp != 0) return cmp;
|
|
}
|
|
for (var i = 0; i < len; i++) {
|
|
final cmp = levels[i].$3.compareTo(other.levels[i].$3);
|
|
if (cmp != 0) return cmp;
|
|
}
|
|
return codepoint.compareTo(other.codepoint);
|
|
}
|
|
}
|
|
|
|
bool _isKatakana(int cp) =>
|
|
(cp >= 0x30A0 && cp <= 0x30FF) || (cp >= 0x31F0 && cp <= 0x31FF) || (cp >= 0xFF65 && cp <= 0xFF9F);
|
|
|
|
Map<int, Weight> parseAllKeys(String text) {
|
|
final result = <int, Weight>{};
|
|
final weightRe = RegExp(r'\[([.*])([0-9A-Fa-f]{4})\.([0-9A-Fa-f]{4})\.([0-9A-Fa-f]{4})\]');
|
|
|
|
for (final line in text.split('\n')) {
|
|
final trimmed = line.trim();
|
|
if (trimmed.isEmpty || trimmed.startsWith('#') || trimmed.startsWith('@')) continue;
|
|
final semiIdx = trimmed.indexOf(';');
|
|
if (semiIdx < 0) continue;
|
|
final cpPart = trimmed.substring(0, semiIdx).trim();
|
|
if (cpPart.contains(' ')) continue;
|
|
final cp = int.tryParse(cpPart, radix: 16);
|
|
if (cp == null || cp > 0xFFFF) continue;
|
|
|
|
final matches = weightRe.allMatches(trimmed.substring(semiIdx + 1));
|
|
if (matches.isEmpty) continue;
|
|
final levels = <(int, int, int)>[];
|
|
for (final match in matches) {
|
|
final primary = int.parse(match.group(2)!, radix: 16);
|
|
final secondary = int.parse(match.group(3)!, radix: 16);
|
|
var tertiary = int.parse(match.group(4)!, radix: 16);
|
|
if (_isKatakana(cp) && tertiary >= 0x000F) tertiary -= 6;
|
|
levels.add((primary, secondary, tertiary));
|
|
}
|
|
if (levels.every((level) => level.$1 == 0 && level.$2 == 0 && level.$3 == 0)) continue;
|
|
if (result.containsKey(cp)) {
|
|
throw FormatException('Duplicate allkeys entry for U+${cp.toRadixString(16).padLeft(4, '0')}');
|
|
}
|
|
result[cp] = Weight(levels, cp);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
(List<int>, Map<int, int>) parseRadicals(String text) {
|
|
final order = <int>[];
|
|
final kangxiDecomp = <int, int>{};
|
|
final radicalRe = RegExp(r'^\[radical \d+');
|
|
|
|
for (final line in text.split('\n')) {
|
|
if (!radicalRe.hasMatch(line)) continue;
|
|
final eqIdx = line.indexOf('=');
|
|
final colonIdx = line.indexOf(':');
|
|
if (eqIdx < 0 || colonIdx < 0 || colonIdx <= eqIdx) continue;
|
|
final closeBracket = line.lastIndexOf(']');
|
|
if (closeBracket < 0) continue;
|
|
|
|
final headerChars = line.substring(eqIdx + 1, colonIdx).runes.toList();
|
|
if (headerChars.length >= 2) {
|
|
final kangxi = headerChars.first;
|
|
final cjk = headerChars[1];
|
|
if (kangxi >= 0x2F00 && kangxi <= 0x2FD5 && cjk <= 0xFFFF) {
|
|
kangxiDecomp[kangxi] = cjk;
|
|
}
|
|
}
|
|
|
|
final runes = line.substring(colonIdx + 1, closeBracket).runes.toList();
|
|
var i = 0;
|
|
while (i < runes.length) {
|
|
final cp = runes[i];
|
|
if (cp == 0x20) {
|
|
i++;
|
|
continue;
|
|
}
|
|
if (i + 2 < runes.length && runes[i + 1] == 0x2D) {
|
|
final endCp = runes[i + 2];
|
|
for (var value = cp; value <= endCp; value++) {
|
|
if (value <= 0xFFFF) order.add(value);
|
|
}
|
|
i += 3;
|
|
continue;
|
|
}
|
|
if (cp <= 0xFFFF) order.add(cp);
|
|
i++;
|
|
}
|
|
}
|
|
|
|
final seen = <int>{};
|
|
return (order.where(seen.add).toList(), kangxiDecomp);
|
|
}
|
|
|
|
List<int> buildOrder(Map<int, Weight> allKeys, List<int> cjkRadicalOrder) {
|
|
final entries = allKeys.entries.toList()..sort((a, b) => a.value.compareTo(b.value));
|
|
final ordered = <int>[];
|
|
final cjkSet = cjkRadicalOrder.toSet();
|
|
var cjkInserted = false;
|
|
|
|
for (final entry in entries) {
|
|
if (cjkSet.contains(entry.key)) continue;
|
|
if (!cjkInserted && entry.value.levels.isNotEmpty && entry.value.levels.first.$1 >= 0xFB00) {
|
|
ordered.addAll(cjkRadicalOrder);
|
|
cjkInserted = true;
|
|
}
|
|
ordered.add(entry.key);
|
|
}
|
|
if (!cjkInserted) ordered.addAll(cjkRadicalOrder);
|
|
return ordered;
|
|
}
|
|
|
|
String buildDenseRankLookup(List<int> ordered) {
|
|
if (ordered.length > 0xFFFF) {
|
|
throw StateError('Rank table overflow: ${ordered.length} entries exceed 65535');
|
|
}
|
|
final ranks = List<int>.filled(0x10000, 0);
|
|
for (var rank = 0; rank < ordered.length; rank++) {
|
|
final codepoint = ordered[rank];
|
|
if (codepoint < 0 || codepoint > 0xFFFF) {
|
|
throw RangeError.range(codepoint, 0, 0xFFFF, 'codepoint');
|
|
}
|
|
if (ranks[codepoint] != 0) {
|
|
throw StateError('Duplicate codepoint U+${codepoint.toRadixString(16).padLeft(4, '0')}');
|
|
}
|
|
ranks[codepoint] = rank + 1;
|
|
}
|
|
return String.fromCharCodes(ranks);
|
|
}
|
|
|
|
String _escapeCodeUnit(int codeUnit) {
|
|
if (codeUnit == 0x27) return r"\'";
|
|
if (codeUnit == 0x5C) return r'\\';
|
|
if (codeUnit == 0x24) return r'\$';
|
|
return '\\u${codeUnit.toRadixString(16).padLeft(4, '0')}';
|
|
}
|
|
|
|
String renderDucetOrder(List<int> ordered, Map<int, int> kangxiDecomp) {
|
|
final denseRanks = buildDenseRankLookup(ordered);
|
|
final buffer = StringBuffer()
|
|
..writeln('/// Dense BMP ranks per DUCET (Unicode 13.0) + pinned CLDR CJK radical-stroke.')
|
|
..writeln('/// Each code unit stores rank + 1; zero means absent.')
|
|
..writeln('/// Katakana sorts before hiragana (CLDR root tailoring).')
|
|
..writeln('/// Generated by scripts/codegen/generate_ducet_ranks.dart — do not edit.')
|
|
..writeln('library;')
|
|
..writeln()
|
|
..writeln('const String _ducetRanks =');
|
|
const unitsPerLine = 128;
|
|
for (var start = 0; start < denseRanks.length; start += unitsPerLine) {
|
|
final end = start + unitsPerLine < denseRanks.length ? start + unitsPerLine : denseRanks.length;
|
|
buffer.write(" '");
|
|
for (var i = start; i < end; i++) {
|
|
buffer.write(_escapeCodeUnit(denseRanks.codeUnitAt(i)));
|
|
}
|
|
buffer.writeln(start + unitsPerLine >= denseRanks.length ? "';" : "'");
|
|
}
|
|
|
|
buffer
|
|
..writeln()
|
|
..writeln('/// Kangxi Radicals (U+2F00-U+2FD5) → CJK Unified equivalents.')
|
|
..writeln('/// ICU decomposes these via NFD before collation.')
|
|
..writeln('const Map<int, int> _kangxiDecomp = {');
|
|
final sortedKangxi = kangxiDecomp.entries.toList()..sort((a, b) => a.key.compareTo(b.key));
|
|
for (final entry in sortedKangxi) {
|
|
buffer.writeln(
|
|
' 0x${entry.key.toRadixString(16).toUpperCase()}: '
|
|
'0x${entry.value.toRadixString(16).toUpperCase()},',
|
|
);
|
|
}
|
|
buffer
|
|
..writeln('};')
|
|
..writeln()
|
|
..writeln('/// Compare two characters using DUCET + CLDR ordering.')
|
|
..writeln('/// Decomposes Kangxi radicals to CJK equivalents before lookup.')
|
|
..writeln('/// Falls back to codepoint order for characters not in the table.')
|
|
..writeln('int ducetCompare(String a, String b) {')
|
|
..writeln(' var cpA = a.runes.first;')
|
|
..writeln(' var cpB = b.runes.first;')
|
|
..writeln(' cpA = _kangxiDecomp[cpA] ?? cpA;')
|
|
..writeln(' cpB = _kangxiDecomp[cpB] ?? cpB;')
|
|
..writeln(' final storedRankA = cpA <= 0xFFFF ? _ducetRanks.codeUnitAt(cpA) : 0;')
|
|
..writeln(' final storedRankB = cpB <= 0xFFFF ? _ducetRanks.codeUnitAt(cpB) : 0;')
|
|
..writeln(' final rankA = storedRankA == 0 ? 0x100000 + cpA : storedRankA - 1;')
|
|
..writeln(' final rankB = storedRankB == 0 ? 0x100000 + cpB : storedRankB - 1;')
|
|
..writeln(' return rankA.compareTo(rankB);')
|
|
..writeln('}');
|
|
return buffer.toString();
|
|
}
|
|
|
|
void writeGeneratedFile(File output, String contents) {
|
|
output.parent.createSync(recursive: true);
|
|
final staged = File('${output.path}.$pid.tmp');
|
|
try {
|
|
staged.writeAsStringSync(contents, flush: true);
|
|
staged.renameSync(output.path);
|
|
} finally {
|
|
if (staged.existsSync()) staged.deleteSync();
|
|
}
|
|
}
|
|
|
|
Future<void> generateDucetRanks({
|
|
bool useBundledSources = true,
|
|
Directory? bundledSourceDirectory,
|
|
File? explicitAllKeys,
|
|
File? explicitFractionalUca,
|
|
Directory? cacheDirectory,
|
|
File? output,
|
|
SourceDownloader downloader = downloadSource,
|
|
GeneratedFileWriter writer = writeGeneratedFile,
|
|
SourceDescriptor allKeysDescriptor = allKeysSource,
|
|
SourceDescriptor fractionalUcaDescriptor = fractionalUcaSource,
|
|
}) async {
|
|
final sources = useBundledSources
|
|
? await loadBundledSources(
|
|
bundledSourceDirectory:
|
|
bundledSourceDirectory ?? Directory.fromUri(Directory.current.uri.resolve('scripts/codegen/data/')),
|
|
allKeysDescriptor: allKeysDescriptor,
|
|
fractionalUcaDescriptor: fractionalUcaDescriptor,
|
|
)
|
|
: await loadVerifiedSources(
|
|
explicitAllKeys: explicitAllKeys,
|
|
explicitFractionalUca: explicitFractionalUca,
|
|
cacheDirectory: cacheDirectory,
|
|
downloader: downloader,
|
|
allKeysDescriptor: allKeysDescriptor,
|
|
fractionalUcaDescriptor: fractionalUcaDescriptor,
|
|
);
|
|
try {
|
|
final allKeys = parseAllKeys(await sources.allKeysFile.readAsString());
|
|
final (cjkOrder, kangxiDecomp) = parseRadicals(await sources.fractionalUcaFile.readAsString());
|
|
final ordered = buildOrder(allKeys, cjkOrder);
|
|
final rendered = renderDucetOrder(ordered, kangxiDecomp);
|
|
sources.commitCaches();
|
|
writer(output ?? File.fromUri(Directory.current.uri.resolve('lib/data/ducet_order.dart')), rendered);
|
|
} finally {
|
|
sources.dispose();
|
|
}
|
|
}
|
|
|
|
Future<void> main(List<String> args) async {
|
|
if (args.length > 2) {
|
|
stderr.writeln('Usage: dart run scripts/codegen/generate_ducet_ranks.dart [allkeys.txt] [FractionalUCA.txt]');
|
|
exitCode = 64;
|
|
return;
|
|
}
|
|
try {
|
|
await generateDucetRanks(
|
|
explicitAllKeys: args.isNotEmpty ? File(args.first) : null,
|
|
explicitFractionalUca: args.length > 1 ? File(args[1]) : null,
|
|
useBundledSources: args.isEmpty,
|
|
);
|
|
stderr.writeln('Written to ${File.fromUri(Directory.current.uri.resolve('lib/data/ducet_order.dart')).path}');
|
|
} catch (error) {
|
|
stderr.writeln(error);
|
|
exitCode = 1;
|
|
}
|
|
}
|