From 6d39c39a94bd89c6bb635031a329fb0bbf8d76ce Mon Sep 17 00:00:00 2001 From: Ben Fisher Date: Sun, 9 Apr 2017 16:20:04 -0700 Subject: [PATCH] restore_from_python.py by Ben Fisher, https://github.com/downpoured a Python script to restore Duplicati files similar to Duplicati.RecoveryTool, but with no dependencies on Mono/.NET uses streaming apis to restore a large number of files and use limited RAM. supports backups using AES encryption (.aes) or No Encryption (.zip), if data uses GPG/other encryption, decrypt files to .zip before running this tool. --- Tools/Commandline/ijson.py | 292 +++++++++++++++++++ Tools/Commandline/pyaescrypt.py | 166 +++++++++++ Tools/Commandline/restore_from_python.py | 347 +++++++++++++++++++++++ 3 files changed, 805 insertions(+) create mode 100644 Tools/Commandline/ijson.py create mode 100644 Tools/Commandline/pyaescrypt.py create mode 100644 Tools/Commandline/restore_from_python.py diff --git a/Tools/Commandline/ijson.py b/Tools/Commandline/ijson.py new file mode 100644 index 000000000..981b08733 --- /dev/null +++ b/Tools/Commandline/ijson.py @@ -0,0 +1,292 @@ +# ijson 2.3 +# Copyright (c) 2010, Ivan Sagalaev +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name "ijson" nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# This software is provided by the regents and contributors ``as is'' and any +# express or implied warranties, including, but not limited to, the implied +# warranties of merchantability and fitness for a particular purpose are +# disclaimed. in no event shall the regents and contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential damages +# (including, but not limited to, procurement of substitute goods or services; +# loss of use, data, or profits; or business interruption) however caused and +# on any theory of liability, whether in contract, strict liability, or tort +# (including negligence or otherwise) arising in any way out of the use of this +# software, even if advised of the possibility of such damage. + +from __future__ import unicode_literals +import decimal +import re +import sys +from codecs import getreader + +class JSONError(Exception): + pass # base exception for all parsing errors. + +class IncompleteJSONError(JSONError): + pass # raised when the parser can't read expected data from a stream. + +class ObjectBuilder(object): + def __init__(self): + def initial_set(value): + self.value = value + self.containers = [initial_set] + + def event(self, event, value): + if event == 'map_key': + self.key = value + elif event == 'start_map': + map = {} + self.containers[-1](map) + def setter(value): + map[self.key] = value + self.containers.append(setter) + elif event == 'start_array': + array = [] + self.containers[-1](array) + self.containers.append(array.append) + elif event == 'end_array' or event == 'end_map': + self.containers.pop() + else: + self.containers[-1](value) + +def parse_impl(basic_events): + path = [] + for event, value in basic_events: + if event == 'map_key': + prefix = '.'.join(path[:-1]) + path[-1] = value + elif event == 'start_map': + prefix = '.'.join(path) + path.append(None) + elif event == 'end_map': + path.pop() + prefix = '.'.join(path) + elif event == 'start_array': + prefix = '.'.join(path) + path.append('item') + elif event == 'end_array': + path.pop() + prefix = '.'.join(path) + else: # any scalar value + prefix = '.'.join(path) + + yield prefix, event, value + +def items_impl(prefixed_events, prefix): + prefixed_events = iter(prefixed_events) + try: + while True: + current, event, value = next(prefixed_events) + if current == prefix: + if event in ('start_map', 'start_array'): + builder = ObjectBuilder() + end_event = event.replace('start', 'end') + while (current, event) != (prefix, end_event): + builder.event(event, value) + current, event, value = next(prefixed_events) + yield builder.value + else: + yield value + except StopIteration: + pass + +def number(str_value): + number = decimal.Decimal(str_value) + int_number = int(number) + if int_number == number: + number = int_number + return number + +class UnexpectedSymbol(JSONError): + def __init__(self, symbol, pos): + super(UnexpectedSymbol, self).__init__( + 'Unexpected symbol %r at %d' % (symbol, pos) + ) + +BUFSIZE = 16 * 1024 +LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S') + +def Lexer(f, buf_size=BUFSIZE): + if type(f.read(0)) == bytetype: + f = getreader('utf-8')(f) + buf = f.read(buf_size) + pos = 0 + discarded = 0 + while True: + match = LEXEME_RE.search(buf, pos) + if match: + lexeme = match.group() + if lexeme == '"': + pos = match.start() + start = pos + 1 + while True: + try: + end = buf.index('"', start) + escpos = end - 1 + while buf[escpos] == '\\': + escpos -= 1 + if (end - escpos) % 2 == 0: + start = end + 1 + else: + break + except ValueError: + data = f.read(buf_size) + if not data: + raise IncompleteJSONError('Incomplete string lexeme') + buf += data + yield discarded + pos, buf[pos:end + 1] + pos = end + 1 + else: + while match.end() == len(buf): + data = f.read(buf_size) + if not data: + break + buf += data + match = LEXEME_RE.search(buf, pos) + lexeme = match.group() + yield discarded + match.start(), lexeme + pos = match.end() + else: + data = f.read(buf_size) + if not data: + break + discarded += len(buf) + buf = data + pos = 0 + +def unescape(s): + start = 0 + result = '' + while start < len(s): + pos = s.find('\\', start) + if pos == -1: + if start == 0: + return s + result += s[start:] + break + result += s[start:pos] + pos += 1 + esc = s[pos] + if esc == 'u': + result += chr(int(s[pos + 1:pos + 5], 16)) + pos += 4 + elif esc == 'b': + result += '\b' + elif esc == 'f': + result += '\f' + elif esc == 'n': + result += '\n' + elif esc == 'r': + result += '\r' + elif esc == 't': + result += '\t' + else: + result += esc + start = pos + 1 + return result + +def parse_value(lexer, symbol=None, pos=0): + try: + if symbol is None: + pos, symbol = next(lexer) + if symbol == 'null': + yield ('null', None) + elif symbol == 'true': + yield ('boolean', True) + elif symbol == 'false': + yield ('boolean', False) + elif symbol == '[': + for event in parse_array(lexer): + yield event + elif symbol == '{': + for event in parse_object(lexer): + yield event + elif symbol[0] == '"': + yield ('string', unescape(symbol[1:-1])) + else: + try: + yield ('number', number(symbol)) + except decimal.InvalidOperation: + raise UnexpectedSymbol(symbol, pos) + except StopIteration: + raise IncompleteJSONError('Incomplete JSON data') + +def parse_array(lexer): + yield ('start_array', None) + try: + pos, symbol = next(lexer) + if symbol != ']': + while True: + for event in parse_value(lexer, symbol, pos): + yield event + pos, symbol = next(lexer) + if symbol == ']': + break + if symbol != ',': + raise UnexpectedSymbol(symbol, pos) + pos, symbol = next(lexer) + yield ('end_array', None) + except StopIteration: + raise IncompleteJSONError('Incomplete JSON data') + +def parse_object(lexer): + yield ('start_map', None) + try: + pos, symbol = next(lexer) + if symbol != '}': + while True: + if symbol[0] != '"': + raise UnexpectedSymbol(symbol, pos) + yield ('map_key', unescape(symbol[1:-1])) + pos, symbol = next(lexer) + if symbol != ':': + raise UnexpectedSymbol(symbol, pos) + for event in parse_value(lexer, None, pos): + yield event + pos, symbol = next(lexer) + if symbol == '}': + break + if symbol != ',': + raise UnexpectedSymbol(symbol, pos) + pos, symbol = next(lexer) + yield ('end_map', None) + except StopIteration: + raise IncompleteJSONError('Incomplete JSON data') + +def basic_parse(file, buf_size=BUFSIZE): + lexer = iter(Lexer(file, buf_size)) + for value in parse_value(lexer): + yield value + try: + next(lexer) + except StopIteration: + pass + else: + raise JSONError('Additional data') + +def parse(file, buf_size=BUFSIZE): + return parse_impl(basic_parse(file, buf_size=buf_size)) + +def items(file, prefix): + return items_impl(parse(file), prefix) + +if sys.version_info[0] < 3: + b2s = lambda s: s + chr = unichr + bytetype = str +else: + def b2s(b): + return b.decode('utf-8') + chr = chr + bytetype = bytes diff --git a/Tools/Commandline/pyaescrypt.py b/Tools/Commandline/pyaescrypt.py new file mode 100644 index 000000000..df8438764 --- /dev/null +++ b/Tools/Commandline/pyaescrypt.py @@ -0,0 +1,166 @@ +# pyAesCrypt 0.1.2 +# Copyright 2016 Marco Bellaccini - marco.bellaccini[at!]gmail.com +# small modifications by Ben Fisher to add fncallback feature +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +def pyAesCryptDecrypt(path, passw, fncallback=None): + from Crypto.Hash import SHA256 + from Crypto.Hash import HMAC + from Crypto.Cipher import AES + + outbytes = b'' + assert path.endswith('.aes'), '%s expected to end with .aes' % path + with open(path, "rb") as fIn: + fdata = fIn.read(3) + # check if file is in AES Crypt format (also min length check) + if fdata != bytes("AES", "utf8") or os.stat(path).st_size < 136: + fail_with_msg("Error: file is corrupted or " + + "not an AES Crypt (or pyAesCrypt) file.") + + # check if file is in AES Crypt format, version 2 + # (the only one compatible with pyAesCrypt) + fdata = fIn.read(1) + if len(fdata) < 1: + fail_with_msg("Error: file is corrupted.") + if fdata != b"\x02": + fail_with_msg("Error: pyAesCrypt is only compatible with version 2 of the " + + "AES Crypt file format.") + + # skip reserved byte + fIn.read(1) + + # skip all the extensions + while True: + fdata = fIn.read(2) + if len(fdata) < 2: + fail_with_msg("Error: file is corrupted.") + if fdata == b"\x00\x00": + break + fIn.read(int.from_bytes(fdata, byteorder="big")) + + # read external iv + iv1 = fIn.read(16) + if len(iv1) < 16: + fail_with_msg("Error: file is corrupted.") + + # stretch password and iv + key=pyAesCryptStretch(passw, iv1) + + # read encrypted main iv and key + c_iv_key = fIn.read(48) + if len(c_iv_key) < 48: + fail_with_msg("Error: file is corrupted.") + + # read HMAC-SHA256 of the encrypted iv and key + hmac1 = fIn.read(32) + if len(hmac1) < 32: + fail_with_msg("Error: file is corrupted.") + + # compute actual HMAC-SHA256 of the encrypted iv and key + hmac1Act = HMAC.new(key, digestmod=SHA256) + hmac1Act.update(c_iv_key) + + # HMAC check + if hmac1 != hmac1Act.digest(): + fail_with_msg("Error: wrong password (or file is corrupted).") + + # instantiate AES cipher + cipher1 = AES.new(key, AES.MODE_CBC, iv1) + + # decrypt main iv and key + iv_key = cipher1.decrypt(c_iv_key) + + # get internal iv and key + iv0 = iv_key[:16] + intKey = iv_key[16:] + + # instantiate another AES cipher + cipher0 = AES.new(intKey, AES.MODE_CBC, iv0) + + # instantiate actual HMAC-SHA256 of the ciphertext + hmac0Act = HMAC.new(intKey, digestmod=SHA256) + + # decrypt ciphertext in large pieces first, then smaller pieces + sizeInputFile = os.stat(path).st_size + for currentBufferSize in [64*64*AES.block_size, 64*AES.block_size, AES.block_size]: + assert 0 == currentBufferSize % AES.block_size + while fIn.tell() < sizeInputFile - 32 - 1 - currentBufferSize: + # read data + cText = fIn.read(currentBufferSize) + # update HMAC + hmac0Act.update(cText) + # decrypt data + if fncallback: + fncallback(cipher0.decrypt(cText)) + else: + outbytes += bytearray(cipher0.decrypt(cText)) + + # last block reached, remove padding if needed + # read last block + if fIn.tell() != os.stat(path).st_size - 32 - 1: # this is for empty files + cText = fIn.read(AES.block_size) + if len(cText) < AES.block_size: + fail_with_msg("Error: file is corrupted.") + else: + cText = bytes() + + # update HMAC + hmac0Act.update(cText) + + # read plaintext file size mod 16 lsb positions + fs16 = fIn.read(1) + if len(fs16) < 1: + fail_with_msg("Error: file is corrupted.") + + # decrypt last block + pText = cipher0.decrypt(cText) + + # remove padding + toremove=((16-fs16[0])%16) + if toremove != 0: + pText=pText[:-toremove] + + if fncallback: + fncallback(pText) + else: + outbytes += bytearray(pText) + + # read HMAC-SHA256 of the encrypted file + hmac0 = fIn.read(32) + if len(hmac0) < 32: + fail_with_msg("Error: file is corrupted.") + + # HMAC check + if hmac0 != hmac0Act.digest(): + fail_with_msg("Error: bad HMAC (file is corrupted).") + return outbytes + +def pyAesCryptStretch(passw, iv1): + # hash the external iv and the password 8192 times + from Crypto.Hash import SHA256 + digest=iv1+(16*b"\x00") + + for i in range(8192): + passHash=SHA256.new() + passHash.update(digest) + passHash.update(bytes(passw,"utf_16_le")) + digest=passHash.digest() + + return digest + +def fail_with_msg(s): + print(s) + raise Exception(s) diff --git a/Tools/Commandline/restore_from_python.py b/Tools/Commandline/restore_from_python.py new file mode 100644 index 000000000..128dded66 --- /dev/null +++ b/Tools/Commandline/restore_from_python.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 + +# by Ben Fisher, https://github.com/downpoured +# a Python script to restore files from Duplicati +# similar to Duplicati.RecoveryTool, but with no dependencies on Mono/.NET +# uses streaming apis to restore a large number of files and use limited RAM. +# supports backups using AES encryption (.aes) or No Encryption (.zip), +# if data uses GPG/other encryption, decrypt files to .zip before running this tool. + +import os +import sys +import io +import json +import ijson +import sqlite3 +import zipfile +import codecs +import getpass +import fnmatch +import base64 +import hashlib +from collections import OrderedDict +from pyaescrypt import pyAesCryptDecrypt, fail_with_msg + +# increase for faster restores, at the cost of higher RAM usage. +maxCacheSizeInMB = 100 + +def mainRestore(dir, outdir, passw, scope): + # locate dlist + dlists = [name for name in os.listdir(dir) if (name.endswith('.dlist.zip') or + name.endswith('.dlist.zip.aes'))] + + if dlists: + dlist = sorted(dlists, reverse=True)[0] + print('using %s which looks like the most recent dlist.' % dlist) + + # decrypt dlist file to disk + if dlist.endswith('.dlist.zip.aes'): + with open(os.path.join(dir, 'py-restore-dlist-decr.zip'), 'wb') as f: + fn = lambda data: f.write(data) + pyAesCryptDecrypt(os.path.join(dir, dlist), passw, fn) + dlist = os.path.join(dir, 'py-restore-dlist-decr.zip') + else: + fail_with_msg('No .dlist.zip files found.') + + # create cache + largestDBlock = max(os.path.getsize(os.path.join(dir, name)) + for name in os.listdir(dir) if '.dblock.zip' in name) + amountInCache = max(1, (maxCacheSizeInMB * 1024 * 1024) // largestDBlock) + cacheDecrypted = MemoizeDecorator(pyAesCryptDecrypt, amountInCache) + + # read some metadata from the manifest + db, numberToName = createDb(dir, 'py-restore-index.sqlite', passw, cacheDecrypted) + dbopts = (db, numberToName, cacheDecrypted, passw) + opts = getArchiveOptions(dir, dlist) + + # restore files + i = 0 + msgs = 0 + print('Restoring files...') + for item in enumerateDlistFiles(dir, dlist): + if item['type'] == 'File' and fnmatch.fnmatch(item['path'], scope): + # print a dot every 10 files to show we're still working + i += 1 + if i % 10 == 0: + sys.stdout.write('.') + sys.stdout.flush() + + if item['path'].startswith('\\\\'): + # windows network share + outPath = outdir + item['path'][1:] + elif item['path'][1:2] == ':' and item['path'][2:3] == '\\': + # windows absolute path + outPath = outdir + '\\' + item['path'][0] + item['path'][2:] + else: + outPath = outdir + item['path'] + + try: + restoreOneFile(dir, dbopts, opts, item, outPath) + except Exception as e: + msgs += 1 + print(toAscii('\nWhen restoring %s to %s: %s' % + (item['path'], outPath, str(e)))) + elif item['type'] == 'Symlink': + print(toAscii('Symlink existed at ' + item['path'])) + + db.close() + print('\n\n%d warnings/errors seen.' % msgs) + +def restoreOneFile(dir, dbopts, opts, listEntry, outPath): + # create destination directory + if not os.path.isdir(os.path.split(outPath)[0]): + os.makedirs(os.path.split(outPath)[0]) + + # write to file + with open(outPath, 'wb') as f: + if 'blocklists' not in listEntry or not listEntry['blocklists']: + # small files store data in one block + data = getContentBlock(dir, dbopts, listEntry['hash']) + f.write(data) + else: + # large files point to a list of blockids, each of which points + # to another list of blockids + for blhi, blh in enumerate(listEntry['blocklists']): + blockhashoffset = blhi * opts['hashes-per-block'] * opts['blocksize'] + binaryHashes = getContentBlock(dir, dbopts, blh) + for bi, start in enumerate(range(0, len(binaryHashes), opts['hash-size'])): + hash = binaryHashes[start: start + opts['hash-size']] + hash = base64.b64encode(hash) + data = getContentBlock(dir, dbopts, hash) + f.seek(blockhashoffset + bi * opts['blocksize']) + f.write(data) + + # verify file size + if listEntry['size'] != os.path.getsize(outPath): + raise Exception('Restored %s. expected filesize %d and got %d' % + (outPath, listEntry['size'], os.path.getsize(outPath))) + + # verify file checksum + hasher = opts['file-hasher']() + computeHash(outPath, hasher) + expected = listEntry['hash'].encode('utf8') + got = base64.b64encode(hasher.digest()) + if expected != got: + raise Exception('Restored %s. expected checksum %s and got %s' % + (outPath, expected, got)) + +def getContentBlock(dir, dbopts, blockId): + if isinstance(blockId, bytes): + blockId = blockId.decode('utf8') + db, numberToName, cacheDecrypted, passw = dbopts + name = getFilenameFromBlockId(db, numberToName, blockId) + with openAsZipFile(dir, name, passw, cacheDecrypted) as zip: + with zip.open(base64PlainToBase64Url(blockId), 'r') as zipContents: + return zipContents.read() + +def openAsZipFile(dir, name, passw, cacheDecrypted): + fullpath = os.path.join(dir, name) + assertTrue(os.path.exists(fullpath), 'missing %s' % fullpath) + if name.endswith('.zip'): + return zipfile.ZipFile(fullpath, 'r') + else: + data = io.BytesIO(cacheDecrypted(fullpath, passw)) + return zipfile.ZipFile(data, 'r') + +def enumerateDlistFiles(dir, dlist): + convertStreamToUtf8 = codecs.getreader('utf-8-sig') + with zipfile.ZipFile(os.path.join(dir, dlist), 'r') as zip: + with zip.open('filelist.json', 'r') as zipentry: + with convertStreamToUtf8(zipentry) as zipentryutf8: + for item in streamJsonArrayItems(zipentryutf8): + yield item + +def streamJsonArrayItems(f): + # read items from a json array -- without loading the entire file into memory + level = 0 + currentObject = ijson.ObjectBuilder() + parsed = ijson.parse(f) + + # eat the initial start_array event + assertEqual('start_array', next(parsed)[1]) + + # construct objects. use level in order to support objects within objects + for path, event, value in parsed: + currentObject.event(event, value) + if event == 'start_map': + level += 1 + elif event == 'end_map': + level -= 1 + if level == 0: + yield currentObject.value + currentObject = ijson.ObjectBuilder() + + # ignore the final end_array event. + +def createDb(dir, filename, passw, cacheDecrypted): + # get a summary of the current dblocks + zipfilenames = [s for s in os.listdir(dir) if + s.endswith('.dblock.zip') or s.endswith('.dblock.zip.aes')] + zipfilenames.sort() + filenamesAndSizes = ';'.join(zipfilenames) + filenamesAndSizes += ';'.join(map(str, + [os.path.getsize(os.path.join(dir, s)) for s in zipfilenames])) + needNew = True + dbpath = os.path.join(dir, filename) + if os.path.exists(dbpath): + # check that the dblocks we have match the dblocks this db has. + dbCheckIfComplete = sqlite3.connect(dbpath) + cursor = dbCheckIfComplete.cursor() + needNew = not cursor.execute('''SELECT FileNum FROM BlockIdToFile + WHERE BlockId=?''', [filenamesAndSizes.encode('utf8')]).fetchone() + cursor.close() + dbCheckIfComplete.close() + + db = sqlite3.connect(dbpath) + cursor = db.cursor() + cursor.execute("PRAGMA temp_store = memory") + cursor.execute("PRAGMA page_size = 16384") + cursor.execute("PRAGMA cache_size = 1000") + cursor.close() + numberToName = OrderedDict((n + 1, v) for n, v in enumerate(zipfilenames)) + if needNew: + print('Creating index, this may take some time...') + createBlockIdsToFilenames(dir, db, passw, cacheDecrypted, + numberToName, filenamesAndSizes) + else: + print('Able to re-use existing index.') + + return db, numberToName + +def createBlockIdsToFilenames(dir, db, passw, cache, numberToName, filenamesAndSizes): + # create an index mapping blockId to filename + with db: + c = db.cursor() + c.execute('''CREATE TABLE IF NOT EXISTS BlockIdToFile ( + BlockId TEXT, + FileNum INTEGER)''') + c.execute('''CREATE INDEX IF NOT EXISTS IxBlockId ON BlockIdToFile(BlockId)''') + c.execute('''DELETE FROM BlockIdToFile WHERE 1''') + for num in numberToName: + name = numberToName[num] + sys.stdout.write('.') + sys.stdout.flush() + with openAsZipFile(dir, name, passw, cache) as zip: + for entryname in zip.namelist(): + entryname = base64UrlToBase64Plain(entryname) + c.execute('INSERT INTO BlockIdToFile (BlockId, FileNum) VALUES (?, ?)', + [entryname.encode('utf8'), num]) + + # write a summary of the current dblocks + c.execute('INSERT INTO BlockIdToFile (BlockId, FileNum) VALUES (?, ?)', + [filenamesAndSizes.encode('utf8'), -1]) + c.close() + db.commit() + + return numberToName + +def base64PlainToBase64Url(data): + if isinstance(data, bytes): return data.replace(b'+', b'-').replace(b'/', b'_') + else: return data.replace('+', '-').replace('/', '_') + +def base64UrlToBase64Plain(data): + if isinstance(data, bytes): return data.replace(b'-', b'+').replace(b'_', b'/') + else: return data.replace('-', '+').replace('_', '/') + +def computeHash(path, hasher, buffersize=64 * 1024): + with open(path, 'rb') as f: + while True: + buffer = f.read(buffersize) + if not buffer: + break + hasher.update(buffer) + +def getFilenameFromBlockId(db, numberToName, blockId): + c = db.cursor() + if isinstance(blockId, str): + blockId = blockId.encode('utf8') + rows = c.execute('SELECT FileNum FROM BlockIdToFile WHERE BlockId=?', [blockId]) + for row in rows: + return numberToName[row[0]] + assertTrue(False, 'block id %s not found' % blockId) + c.close() + +def toAscii(s): + import unicodedata + s = unicodedata.normalize('NFKD', str(s)) + return s.encode('ascii', 'ignore').decode('ascii') + +def assertEqual(v, expect, context=''): + if v != expect: + s = 'Not equal: ' + context + ' Expected ' + expect + ' but got ' + v + raise AssertionError(toAscii(s)) + +def assertTrue(condition, *context): + if not condition: + s = ' '.join(context) if context else '' + raise AssertionError(toAscii(s)) + +# code.activestate.com/recipes/496879-memoize-decorator-function-with-cache-size-limit/ +def MemoizeDecorator(fn, cachesize): + cache = OrderedDict() + def memoize_wrapper(*args, **kwargs): + import pickle + key = pickle.dumps((args, kwargs)) + try: + return cache[key] + except KeyError: + result = fn(*args, **kwargs) + cache[key] = result + if len(cache) > memoize_wrapper._limit: + # remove like in a FIFO queue + cache.popitem(False) + return result + + memoize_wrapper._limit = cachesize + memoize_wrapper._cache = cache + return memoize_wrapper + +def getHasherObject(hashalg): + hashalg = hashalg.lower() + if hashalg == 'sha1': return lambda: hashlib.sha1() + elif hashalg == 'md5': return lambda: hashlib.md5() + elif hashalg == 'sha256': return lambda: hashlib.sha256() + elif hashalg == 'sha384': return lambda: hashlib.sha384() + elif hashalg == 'sha512': return lambda: hashlib.sha512() + else: assertTrue(False, 'unknown hash algorithm %s' % hashalg) + +def getArchiveOptions(dir, dlist): + opts = {} + convertStreamToUtf8 = codecs.getreader('utf-8-sig') + with zipfile.ZipFile(os.path.join(dir, dlist), 'r') as zip: + with zip.open('manifest', 'r') as zipentry: + with convertStreamToUtf8(zipentry) as zipentryutf8: + alljson = zipentryutf8.read() + manifest = json.loads(alljson) + assertEqual(manifest['BlockHash'], manifest['FileHash'], + 'script currently needs same hash method for blockhash and filehash') + opts['blocksize'] = int(manifest['Blocksize']) + opts['block-hasher'] = getHasherObject(manifest['BlockHash']) + opts['file-hasher'] = getHasherObject(manifest['FileHash']) + opts['hash-size'] = opts['block-hasher']().digest_size + opts['hashes-per-block'] = (opts['blocksize'] // + (opts['block-hasher']().digest_size)) + return opts + +def main(): + print('Welcome to Python Duplicati recovery.') + dir = input('Please type the full path to a directory with Duplicati\'s .aes or .zip files:') + assertTrue(os.path.isdir(dir), 'Directory not found') + scope = input('Please type * to restore all files, or a pattern like /path/to/files/* to ' + + 'restore the files in a certain directory)') + outdir = input('Please enter the path to an empty destination directory:') + assertTrue(os.path.isdir(outdir), 'Output directory not found') + assertTrue(len(os.listdir(outdir)) == 0, 'Output directory not empty') + if sys.platform.startswith('win') and len(outdir) > 40: + print('note: paths on windows have limited length, you might want to consider a shorter output path.') + + # get password + passw = None + if any(name.endswith('.aes') for name in os.listdir(dir)): + passw = str(getpass.getpass("Password:")) + + mainRestore(dir, outdir, passw, scope) + print('Complete.') + +if __name__ == '__main__': + main()