haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3092 from piegamesde/master

Browse Source 
Made python script run in parallel
This commit is contained in:
Kenneth Skovhede
2018-03-16 08:40:51 +01:00
committed by GitHub
parent e124329f1e 77a46b34c0
commit 9f63caae4e
2 changed files with 39 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/Commandline/ReEncrypt/ReEncrypt.py
+36 -31
View File
@@ -28,6 +28,8 @@ from collections import OrderedDict
from tempfile import mkstemp, mkdtemp, TemporaryFile, TemporaryDirectory, NamedTemporaryFile
import gnupg
import shutil
from joblib import Parallel, delayed
import multiprocessing
def mainReEncrypt(options):
# locate dlist
@@ -44,48 +46,51 @@ def mainReEncrypt(options):
# locate dlist
dindex = [name for name in os.listdir(options['orig']['path']) if name.endswith(".dindex.%s" %(options['orig']['extension']))]
num_cores = multiprocessing.cpu_count()
Parallel(n_jobs=num_cores*2)(delayed(handleIndex)(options, dindex_enc) for dindex_enc in dindex)
def handleIndex(options, dindex_enc):
with NamedTemporaryFile() as temp_dindex, NamedTemporaryFile() as temp_dindex_reenc, TemporaryDirectory() as temp_path_zip:
for dindex_enc in dindex:
dindex_enc_fullpath = os.path.join(options['orig']['path'],dindex_enc)
dindex_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension']))
dindex_enc_fullpath = os.path.join(options['orig']['path'],dindex_enc)
dindex_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension']))
decrypt(options['orig'],dindex_enc_fullpath,options['orig']['passwd'],temp_dindex.name)
decrypt(options['orig'],dindex_enc_fullpath,options['orig']['passwd'],temp_dindex.name)
unzip(temp_dindex,temp_path_zip)
vol_path = os.path.join(temp_path_zip,'vol')
unzip(temp_dindex,temp_path_zip)
for dblock in os.listdir(vol_path):
data = []
with open(os.path.join(vol_path, dblock)) as data_file:
data = json.load(data_file, object_pairs_hook=OrderedDict)
vol_path = os.path.join(temp_path_zip,'vol')
for dblock in os.listdir(vol_path):
data = []
with open(os.path.join(vol_path, dblock)) as data_file:
data = json.load(data_file, object_pairs_hook=OrderedDict)
expected_hash = data['volumehash'].encode('utf8')
expected_volumesize = data['volumesize']
expected_hash = data['volumehash'].encode('utf8')
expected_volumesize = data['volumesize']
if (options['verify_hash']):
actual_hash = computeHash(os.path.join(options['orig']['path'],dblock))
actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size
print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash))
if (options['verify_hash']):
actual_hash = computeHash(os.path.join(options['orig']['path'],dblock))
actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size
print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash))
with NamedTemporaryFile() as temp_dblock:
dblock_enc_fullpath = os.path.join(options['orig']['path'],dblock)
dblock_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))
decrypt(options['orig'],dblock_enc_fullpath,options['orig']['passwd'],temp_dblock.name)
encrypt(options['new'],temp_dblock.name,options['new']['passwd'], dblock_reenc_fullpath)
with NamedTemporaryFile() as temp_dblock:
dblock_enc_fullpath = os.path.join(options['orig']['path'],dblock)
dblock_reenc_fullpath = os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))
decrypt(options['orig'],dblock_enc_fullpath,options['orig']['passwd'],temp_dblock.name)
encrypt(options['new'],temp_dblock.name,options['new']['passwd'], dblock_reenc_fullpath)
new_hash = computeHash(dblock_reenc_fullpath)
data['volumehash'] = new_hash.decode('utf8')
data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size
print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash']))
data['volumehash'] = new_hash.decode('utf8')
data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size
print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash']))
with open(os.path.join(vol_path,dblock),'w') as data_file:
json.dump(data, data_file)
with open(os.path.join(vol_path,dblock),'w') as data_file:
json.dump(data, data_file)
os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension'])))
os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension'])))
make_zipfile(temp_dindex_reenc.name,temp_path_zip)
encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],dindex_reenc_fullpath)
make_zipfile(temp_dindex_reenc.name,temp_path_zip)
encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],dindex_reenc_fullpath)
def change_ext(filename, ext_old, ext_new):
return filename.replace(ext_old, ext_new)
Tools/Commandline/ReEncrypt/Readme
+3 -3
View File
@@ -6,12 +6,12 @@ Usage:
1) Confirm that your Duplicati data is on disk in .zip or .zip.aes or .zip.gpg format..
2) Install Python 3 if it is not already installed. pyAesCrypt and gnupg packages need to be installed in python as well.
2) Install Python 3 if it is not already installed. pyAesCrypt, gnupg and joblib packages need to be installed in python as well.
3) Prepare a config file (example config.txt).
3) Prepare a config file (see example config.txt).
3) Run ReEncrypt.py -c config.txt.
4) Update your backup settings to the changed encryption settings.
5) Take a backup of your database and do a delete & recreate.
5) Take a backup of your database and do a delete & recreate. Don't forget to delete, a simple "repair" may delete the whole backup!