579 lines
20 KiB
Python
579 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script to generate dlist files from Duplicati SQLite database.
|
|
|
|
This script reads from a Duplicati local backup database and generates
|
|
dlist (fileset) files for each backup version found.
|
|
|
|
Usage:
|
|
python generate_dlist_files.py <database_path> [options]
|
|
|
|
Example:
|
|
python generate_dlist_files.py /path/to/backup.sqlite --output-dir ./dlist_files
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import zipfile
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
|
|
# Constants matching Duplicati's format
|
|
SERIALIZED_DATE_TIME_FORMAT = "%Y%m%dT%H%M%SZ"
|
|
FILELIST_FILENAME = "filelist.json"
|
|
FILESET_FILENAME = "fileset"
|
|
MANIFEST_FILENAME = "manifest"
|
|
MANIFEST_VERSION = 2
|
|
MANIFEST_ENCODING = "utf8"
|
|
|
|
# Special blockset IDs
|
|
FOLDER_BLOCKSET_ID = -100
|
|
SYMLINK_BLOCKSET_ID = -200
|
|
|
|
# Unix epoch starts at 1970-01-01, .NET ticks start at 0001-01-01
|
|
# The difference is 621355968000000000 ticks
|
|
TICKS_TO_1970 = 621355968000000000
|
|
TICKS_PER_SECOND = 10000000
|
|
|
|
|
|
def convert_timestamp(ts: int) -> datetime:
|
|
"""
|
|
Convert a Duplicati timestamp to Python datetime.
|
|
Duplicati may store timestamps as:
|
|
- Unix seconds (values around 1-2 billion for dates in 2020s)
|
|
- Unix milliseconds (values around 1-2 trillion)
|
|
- .NET DateTime ticks (values around 6-7 quadrillion for dates in 2020s)
|
|
"""
|
|
if ts is None:
|
|
return datetime.now(timezone.utc)
|
|
|
|
# Detect format based on magnitude
|
|
if ts > TICKS_TO_1970:
|
|
# .NET DateTime ticks (100-nanosecond intervals since 0001-01-01)
|
|
unix_seconds = (ts - TICKS_TO_1970) / TICKS_PER_SECOND
|
|
return datetime.fromtimestamp(unix_seconds, tz=timezone.utc)
|
|
elif ts > 1000000000000: # 1 trillion
|
|
# Likely Unix milliseconds
|
|
return datetime.fromtimestamp(ts / 1000, tz=timezone.utc)
|
|
else:
|
|
# Likely Unix seconds
|
|
return datetime.fromtimestamp(ts, tz=timezone.utc)
|
|
|
|
|
|
def serialize_datetime(dt: datetime) -> str:
|
|
"""Serialize datetime to Duplicati format (UTC)."""
|
|
return dt.astimezone(timezone.utc).strftime(SERIALIZED_DATE_TIME_FORMAT)
|
|
|
|
|
|
def deserialize_datetime(s: str) -> datetime:
|
|
"""Deserialize datetime from Duplicati format."""
|
|
return datetime.strptime(s, SERIALIZED_DATE_TIME_FORMAT).replace(tzinfo=timezone.utc)
|
|
|
|
|
|
def generate_filename(
|
|
filetype: str,
|
|
prefix: str,
|
|
timestamp: datetime,
|
|
compression_module: str = "zip",
|
|
encryption_module: Optional[str] = None
|
|
) -> str:
|
|
"""Generate a dlist filename matching Duplicati's naming convention."""
|
|
volumename = f"{prefix}-{serialize_datetime(timestamp)}.{filetype}.{compression_module}"
|
|
if encryption_module:
|
|
volumename += f".{encryption_module}"
|
|
return volumename
|
|
|
|
|
|
def create_manifest_data(blocksize: int, blockhash: str, filehash: str) -> Dict[str, Any]:
|
|
"""Create manifest data matching Duplicati's format."""
|
|
return {
|
|
"Version": MANIFEST_VERSION,
|
|
"Created": serialize_datetime(datetime.now(timezone.utc)),
|
|
"Encoding": MANIFEST_ENCODING,
|
|
"Blocksize": blocksize,
|
|
"BlockHash": blockhash,
|
|
"FileHash": filehash,
|
|
"AppVersion": "2.0.0.0" # Placeholder
|
|
}
|
|
|
|
|
|
def create_fileset_data(is_full_backup: bool) -> Dict[str, Any]:
|
|
"""Create fileset data matching Duplicati's format."""
|
|
return {
|
|
"IsFullBackup": is_full_backup
|
|
}
|
|
|
|
|
|
class DuplicatiDatabase:
|
|
"""Wrapper for Duplicati SQLite database access."""
|
|
|
|
def __init__(self, db_path: str):
|
|
self.db_path = db_path
|
|
self.conn = sqlite3.connect(db_path)
|
|
self.conn.row_factory = sqlite3.Row
|
|
|
|
def close(self):
|
|
"""Close database connection."""
|
|
self.conn.close()
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.close()
|
|
|
|
def get_configuration(self) -> Dict[str, str]:
|
|
"""Get configuration settings from the database."""
|
|
cursor = self.conn.execute('SELECT "Key", "Value" FROM "Configuration"')
|
|
return {row["Key"]: row["Value"] for row in cursor.fetchall()}
|
|
|
|
def get_filesets(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get all filesets (backup versions) from the database.
|
|
|
|
Returns list of dicts with: id, operation_id, volume_id, is_full_backup, timestamp
|
|
"""
|
|
cursor = self.conn.execute('''
|
|
SELECT
|
|
"ID" as id,
|
|
"OperationID" as operation_id,
|
|
"VolumeID" as volume_id,
|
|
"IsFullBackup" as is_full_backup,
|
|
"Timestamp" as timestamp
|
|
FROM "Fileset"
|
|
ORDER BY "Timestamp" DESC
|
|
''')
|
|
|
|
filesets = []
|
|
for row in cursor.fetchall():
|
|
filesets.append({
|
|
"id": row["id"],
|
|
"operation_id": row["operation_id"],
|
|
"volume_id": row["volume_id"],
|
|
"is_full_backup": bool(row["is_full_backup"]),
|
|
"timestamp": convert_timestamp(row["timestamp"])
|
|
})
|
|
return filesets
|
|
|
|
def get_files_for_fileset(self, fileset_id: int) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get all files belonging to a specific fileset.
|
|
|
|
Returns list of file entries with all metadata needed for dlist.
|
|
"""
|
|
cursor = self.conn.execute('''
|
|
SELECT
|
|
f."Path" as path,
|
|
fe."Lastmodified" as lastmodified,
|
|
fl."BlocksetID" as blockset_id,
|
|
fl."MetadataID" as metadata_id,
|
|
CASE
|
|
WHEN fl."BlocksetID" = ? THEN 'Folder'
|
|
WHEN fl."BlocksetID" = ? THEN 'Symlink'
|
|
ELSE 'File'
|
|
END as entry_type
|
|
FROM "FilesetEntry" fe
|
|
JOIN "File" f ON fe."FileID" = f."ID"
|
|
JOIN "FileLookup" fl ON f."ID" = fl."ID"
|
|
WHERE fe."FilesetID" = ?
|
|
ORDER BY f."Path"
|
|
''', (FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, fileset_id))
|
|
|
|
files = []
|
|
for row in cursor.fetchall():
|
|
entry = {
|
|
"path": row["path"],
|
|
"lastmodified": convert_timestamp(row["lastmodified"]),
|
|
"blockset_id": row["blockset_id"],
|
|
"metadata_id": row["metadata_id"],
|
|
"type": row["entry_type"]
|
|
}
|
|
|
|
# Get file content info (for files and alternate streams)
|
|
if entry["type"] == "File":
|
|
content_info = self._get_blockset_info(entry["blockset_id"])
|
|
if content_info:
|
|
entry.update(content_info)
|
|
|
|
# Get metadata info
|
|
metadata_info = self._get_metadata_info(entry["metadata_id"])
|
|
if metadata_info:
|
|
entry["metadata"] = metadata_info
|
|
|
|
files.append(entry)
|
|
|
|
return files
|
|
|
|
def _get_blockset_info(self, blockset_id: int) -> Optional[Dict[str, Any]]:
|
|
"""Get blockset information (hash, size, blocklists)."""
|
|
if blockset_id is None or blockset_id <= 0:
|
|
return None
|
|
|
|
cursor = self.conn.execute('''
|
|
SELECT "Length" as length, "FullHash" as fullhash
|
|
FROM "Blockset"
|
|
WHERE "ID" = ?
|
|
''', (blockset_id,))
|
|
|
|
row = cursor.fetchone()
|
|
if not row:
|
|
return None
|
|
|
|
result = {
|
|
"size": row["length"],
|
|
"hash": row["fullhash"]
|
|
}
|
|
|
|
# Get blocklist hashes for large files (files with multiple blocks)
|
|
cursor = self.conn.execute('''
|
|
SELECT "Hash" as hash
|
|
FROM "BlocklistHash"
|
|
WHERE "BlocksetID" = ?
|
|
ORDER BY "Index"
|
|
''', (blockset_id,))
|
|
|
|
blocklist_hashes = [r["hash"] for r in cursor.fetchall()]
|
|
if blocklist_hashes:
|
|
result["blocklists"] = blocklist_hashes
|
|
else:
|
|
# Small file stored as single block - get block hash directly
|
|
cursor = self.conn.execute('''
|
|
SELECT b."Hash" as hash, b."Size" as size
|
|
FROM "BlocksetEntry" be
|
|
JOIN "Block" b ON be."BlockID" = b."ID"
|
|
WHERE be."BlocksetID" = ?
|
|
ORDER BY be."Index"
|
|
LIMIT 1
|
|
''', (blockset_id,))
|
|
|
|
block_row = cursor.fetchone()
|
|
if block_row:
|
|
result["blockhash"] = block_row["hash"]
|
|
result["blocksize"] = block_row["size"]
|
|
|
|
return result
|
|
|
|
def _get_metadata_info(self, metadata_id: int) -> Optional[Dict[str, Any]]:
|
|
"""Get metadata blockset information."""
|
|
if metadata_id is None or metadata_id <= 0:
|
|
return None
|
|
|
|
cursor = self.conn.execute('''
|
|
SELECT "BlocksetID" as blockset_id, "Content" as content
|
|
FROM "Metadataset"
|
|
WHERE "ID" = ?
|
|
''', (metadata_id,))
|
|
|
|
row = cursor.fetchone()
|
|
if not row:
|
|
return None
|
|
|
|
result = {}
|
|
|
|
# Get metadata blockset info
|
|
if row["blockset_id"]:
|
|
cursor = self.conn.execute('''
|
|
SELECT "Length" as length, "FullHash" as fullhash
|
|
FROM "Blockset"
|
|
WHERE "ID" = ?
|
|
''', (row["blockset_id"],))
|
|
|
|
bs_row = cursor.fetchone()
|
|
if bs_row:
|
|
result["metasize"] = bs_row["length"]
|
|
result["metahash"] = bs_row["fullhash"]
|
|
|
|
# Get metablocklist hashes
|
|
cursor = self.conn.execute('''
|
|
SELECT "Hash" as hash
|
|
FROM "BlocklistHash"
|
|
WHERE "BlocksetID" = ?
|
|
ORDER BY "Index"
|
|
''', (row["blockset_id"],))
|
|
|
|
metablocklist_hashes = [r["hash"] for r in cursor.fetchall()]
|
|
if metablocklist_hashes:
|
|
result["metablocklists"] = metablocklist_hashes
|
|
else:
|
|
# Small metadata stored as single block
|
|
cursor = self.conn.execute('''
|
|
SELECT b."Hash" as hash
|
|
FROM "BlocksetEntry" be
|
|
JOIN "Block" b ON be."BlockID" = b."ID"
|
|
WHERE be."BlocksetID" = ?
|
|
ORDER BY be."Index"
|
|
LIMIT 1
|
|
''', (row["blockset_id"],))
|
|
|
|
meta_block_row = cursor.fetchone()
|
|
if meta_block_row:
|
|
result["metablockhash"] = meta_block_row["hash"]
|
|
|
|
return result if result else None
|
|
|
|
|
|
def create_filelist_json(files: List[Dict[str, Any]]) -> str:
|
|
"""
|
|
Create the filelist.json content as a JSON array string.
|
|
|
|
Each file entry follows Duplicati's format:
|
|
{
|
|
"type": "File" | "Folder" | "Symlink" | "AlternateStream",
|
|
"path": "/path/to/file",
|
|
"hash": "abc123...",
|
|
"size": 12345,
|
|
"time": "20240115T120000Z",
|
|
"metahash": "def456...",
|
|
"metasize": 123,
|
|
"metablockhash": "ghi789..." | "metablocklists": ["hash1", "hash2"],
|
|
"blockhash": "jkl012...",
|
|
"blocksize": 102400,
|
|
"blocklists": ["hash1", "hash2"]
|
|
}
|
|
"""
|
|
entries = []
|
|
|
|
for file_info in files:
|
|
entry = {
|
|
"type": file_info["type"],
|
|
"path": file_info["path"]
|
|
}
|
|
|
|
# Add file-specific fields
|
|
if file_info["type"] in ("File", "AlternateStream"):
|
|
if "hash" in file_info:
|
|
entry["hash"] = file_info["hash"]
|
|
if "size" in file_info:
|
|
entry["size"] = file_info["size"]
|
|
if "lastmodified" in file_info:
|
|
entry["time"] = serialize_datetime(file_info["lastmodified"])
|
|
|
|
# Add block info (either blocklists for large files or single blockhash)
|
|
if "blocklists" in file_info:
|
|
entry["blocklists"] = file_info["blocklists"]
|
|
elif "blockhash" in file_info:
|
|
entry["blockhash"] = file_info["blockhash"]
|
|
entry["blocksize"] = file_info.get("blocksize", 0)
|
|
|
|
# Add metadata fields
|
|
if "metadata" in file_info and file_info["metadata"]:
|
|
meta = file_info["metadata"]
|
|
if "metahash" in meta:
|
|
entry["metahash"] = meta["metahash"]
|
|
if "metasize" in meta:
|
|
entry["metasize"] = meta["metasize"]
|
|
if "metablocklists" in meta:
|
|
entry["metablocklists"] = meta["metablocklists"]
|
|
elif "metablockhash" in meta:
|
|
entry["metablockhash"] = meta["metablockhash"]
|
|
|
|
entries.append(entry)
|
|
|
|
# Return as JSON array (without newlines between entries to match Duplicati's compact format)
|
|
return json.dumps(entries, separators=(',', ':'))
|
|
|
|
|
|
def generate_dlist_file(
|
|
output_path: str,
|
|
fileset: Dict[str, Any],
|
|
files: List[Dict[str, Any]],
|
|
config: Dict[str, str],
|
|
compression: str = "zip",
|
|
blocksize_override: Optional[int] = None,
|
|
blockhash_override: Optional[str] = None,
|
|
filehash_override: Optional[str] = None,
|
|
prefix_override: Optional[str] = None
|
|
) -> str:
|
|
"""
|
|
Generate a dlist file for a specific fileset.
|
|
|
|
Args:
|
|
output_path: Directory where the file will be written
|
|
fileset: Fileset metadata (id, timestamp, is_full_backup, etc.)
|
|
files: List of file entries for this fileset
|
|
config: Database configuration (blocksize, blockhash, filehash)
|
|
compression: Compression module to use (default: zip)
|
|
blocksize_override: Optional override for blocksize from command line
|
|
blockhash_override: Optional override for blockhash from command line
|
|
filehash_override: Optional override for filehash from command line
|
|
prefix_override: Optional override for prefix from command line
|
|
|
|
Returns:
|
|
Path to the generated file
|
|
"""
|
|
# Get configuration values with defaults, allowing command-line overrides
|
|
blocksize = blocksize_override if blocksize_override is not None else int(config.get("blocksize", "102400"))
|
|
blockhash = blockhash_override if blockhash_override is not None else config.get("block-hash-algorithm", "SHA256")
|
|
filehash = filehash_override if filehash_override is not None else config.get("file-hash-algorithm", "SHA256")
|
|
prefix = prefix_override if prefix_override is not None else config.get("prefix", "duplicati")
|
|
|
|
# Generate filename
|
|
filename = generate_filename(
|
|
"dlist",
|
|
prefix,
|
|
fileset["timestamp"],
|
|
compression
|
|
)
|
|
|
|
filepath = os.path.join(output_path, filename)
|
|
|
|
# Create the zip file
|
|
with zipfile.ZipFile(filepath, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
|
|
# Add manifest file
|
|
manifest_data = create_manifest_data(blocksize, blockhash, filehash)
|
|
zf.writestr(MANIFEST_FILENAME, json.dumps(manifest_data))
|
|
|
|
# Add fileset file
|
|
fileset_data = create_fileset_data(fileset["is_full_backup"])
|
|
zf.writestr(FILESET_FILENAME, json.dumps(fileset_data))
|
|
|
|
# Add filelist.json
|
|
filelist_json = create_filelist_json(files)
|
|
zf.writestr(FILELIST_FILENAME, filelist_json)
|
|
|
|
return filepath
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Generate dlist files from Duplicati SQLite database"
|
|
)
|
|
parser.add_argument(
|
|
"database",
|
|
help="Path to the Duplicati SQLite database file"
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir", "-o",
|
|
default=".",
|
|
help="Output directory for generated dlist files (default: current directory)"
|
|
)
|
|
parser.add_argument(
|
|
"--compression", "-c",
|
|
default="zip",
|
|
help="Compression format: zip (default)"
|
|
)
|
|
parser.add_argument(
|
|
"--fileset-id", "-f",
|
|
type=int,
|
|
help="Generate only for specific fileset ID (default: all filesets)"
|
|
)
|
|
parser.add_argument(
|
|
"--list", "-l",
|
|
action="store_true",
|
|
help="List filesets without generating files"
|
|
)
|
|
parser.add_argument(
|
|
"--blocksize", "-b",
|
|
type=int,
|
|
help="Block size in bytes (default: read from database, or 102400)"
|
|
)
|
|
parser.add_argument(
|
|
"--blockhash",
|
|
help="Block hash algorithm (default: read from database, or SHA256)"
|
|
)
|
|
parser.add_argument(
|
|
"--filehash",
|
|
help="File hash algorithm (default: read from database, or SHA256)"
|
|
)
|
|
parser.add_argument(
|
|
"--prefix", "-p",
|
|
help="Filename prefix (default: read from database, or 'duplicati')"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Validate database path
|
|
if not os.path.exists(args.database):
|
|
print(f"Error: Database file not found: {args.database}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
# Create output directory if needed
|
|
if not args.list and not os.path.exists(args.output_dir):
|
|
os.makedirs(args.output_dir)
|
|
|
|
# Open database
|
|
with DuplicatiDatabase(args.database) as db:
|
|
# Get configuration
|
|
config = db.get_configuration()
|
|
|
|
# Get all filesets
|
|
filesets = db.get_filesets()
|
|
|
|
if not filesets:
|
|
print("No filesets found in database.")
|
|
sys.exit(0)
|
|
|
|
# List mode
|
|
if args.list:
|
|
print(f"\nFound {len(filesets)} fileset(s) in database:")
|
|
print("-" * 80)
|
|
for fs in filesets:
|
|
backup_type = "FULL" if fs["is_full_backup"] else "PARTIAL"
|
|
print(f" ID: {fs['id']}")
|
|
print(f" Timestamp: {fs['timestamp'].isoformat()}")
|
|
print(f" Type: {backup_type}")
|
|
print(f" Volume ID: {fs['volume_id']}")
|
|
print("-" * 80)
|
|
return
|
|
|
|
# Filter to specific fileset if requested
|
|
if args.fileset_id is not None:
|
|
filesets = [fs for fs in filesets if fs["id"] == args.fileset_id]
|
|
if not filesets:
|
|
print(f"Error: Fileset ID {args.fileset_id} not found", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
# Show configuration being used
|
|
print(f"\nGenerating dlist files for {len(filesets)} fileset(s)...")
|
|
print(f"Output directory: {os.path.abspath(args.output_dir)}")
|
|
print("-" * 80)
|
|
print("Configuration:")
|
|
blocksize = args.blocksize if args.blocksize is not None else int(config.get("blocksize", "102400"))
|
|
blockhash = args.blockhash if args.blockhash is not None else config.get("block-hash-algorithm", "SHA256")
|
|
filehash = args.filehash if args.filehash is not None else config.get("file-hash-algorithm", "SHA256")
|
|
prefix = args.prefix if args.prefix is not None else config.get("prefix", "duplicati")
|
|
print(f" Blocksize: {blocksize} bytes {'(from command line)' if args.blocksize else '(from database)'}")
|
|
print(f" Blockhash: {blockhash} {'(from command line)' if args.blockhash else '(from database)'}")
|
|
print(f" Filehash: {filehash} {'(from command line)' if args.filehash else '(from database)'}")
|
|
print(f" Prefix: {prefix} {'(from command line)' if args.prefix else '(from database)'}")
|
|
print("-" * 80)
|
|
|
|
generated_files = []
|
|
for fileset in filesets:
|
|
print(f"\nProcessing fileset ID {fileset['id']} ({fileset['timestamp'].isoformat()})...")
|
|
|
|
# Get files for this fileset
|
|
files = db.get_files_for_fileset(fileset["id"])
|
|
print(f" Found {len(files)} files")
|
|
|
|
# Generate dlist file
|
|
filepath = generate_dlist_file(
|
|
args.output_dir,
|
|
fileset,
|
|
files,
|
|
config,
|
|
args.compression,
|
|
blocksize_override=args.blocksize,
|
|
blockhash_override=args.blockhash,
|
|
filehash_override=args.filehash,
|
|
prefix_override=args.prefix
|
|
)
|
|
|
|
generated_files.append(filepath)
|
|
file_size = os.path.getsize(filepath)
|
|
print(f" Generated: {os.path.basename(filepath)}")
|
|
print(f" Size: {file_size:,} bytes")
|
|
|
|
print("\n" + "=" * 80)
|
|
print(f"Successfully generated {len(generated_files)} dlist file(s):")
|
|
for f in generated_files:
|
|
print(f" - {f}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|