haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6898 from duplicati/feature/handle-empty-index-files

Browse Source 
Handle empty index files
This commit is contained in:
Kenneth Skovhede
2026-05-08 10:31:55 +02:00
committed by GitHub
parent 4afa374d7b 95516aa08b
commit a1968ed515
3 changed files with 306 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Volumes/IndexVolumeReader.cs
+48 -21
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
@@ -136,8 +136,29 @@ namespace Duplicati.Library.Main.Volumes
while (this.MoveNext())
{ /*skip*/ }
var hash = ReadJsonStringProperty(m_reader, "volumehash");
var size = ReadJsonInt64Property(m_reader, "volumesize");
if (!m_reader.Read())
throw new InvalidDataException("Invalid JSON, EOF found while reading volume properties");
string hash = null;
long size = 0;
if (m_reader.TokenType == JsonToken.PropertyName)
{
if (m_reader.Value?.ToString() != "volumehash")
throw new InvalidDataException(string.Format("Invalid JSON, expected property \"volumehash\", but got {0}, {1}", m_reader.TokenType, m_reader.Value));
if (!m_reader.Read() || m_reader.TokenType != JsonToken.String)
throw new InvalidDataException(string.Format("Invalid JSON, expected String, but got {0}, {1}", m_reader.TokenType, m_reader.Value));
hash = m_reader.Value?.ToString();
size = ReadJsonInt64Property(m_reader, "volumesize");
}
// Handle EndObject because specail cases could write an empty array: {"blocks":[]}
// In this case, there are no volume properties, so we just return an empty value
else if (m_reader.TokenType != JsonToken.EndObject)
{
throw new InvalidDataException(string.Format("Invalid JSON, expected PropertyName or EndObject, but got {0}, {1}", m_reader.TokenType, m_reader.Value));
}
return (m_volumeProps = new KeyValuePair<string, long>(hash, size)).Value;
}
@@ -284,6 +305,9 @@ namespace Duplicati.Library.Main.Volumes
while (m_index < m_files.Length && ParseFilename(m_files[m_index]) == null)
m_index++;
if (m_index >= m_files.Length)
return false;
m_current = new IndexBlockVolume(m_compression, m_files[m_index]);
return true;
@@ -393,6 +417,9 @@ namespace Duplicati.Library.Main.Volumes
while (m_index < m_files.Length && IsValidBase64Hash(m_files[m_index].Key, m_hashsize))
m_index++;
if (m_index >= m_files.Length)
return false;
m_current = new IndexBlocklist(m_compression, m_files[m_index].Key, m_files[m_index].Value, m_hashsize, m_blockHashAlgorithm);
return true;
Duplicati/Library/Main/Volumes/VolumeReaderBase.cs
+3
View File
@@ -226,6 +226,9 @@ namespace Duplicati.Library.Main.Volumes
using var hashalg = HashFactory.CreateHasher(blockHashAlgorithm);
using var fs = compression.OpenRead(filename);
if (fs == null)
throw new InvalidDataException($"Blocklist file {filename} not found in archive");
int s;
var read = 0L;
while ((s = Library.Utility.Utility.ForceStreamRead(fs, buffer, buffer.Length)) != 0)
Duplicati/UnitTest/Issue6892.cs
+255
View File
@@ -0,0 +1,255 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.IO;
using System.Linq;
using NUnit.Framework;
using Duplicati.Library.Main.Volumes;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
namespace Duplicati.UnitTest;
public class Issue6892 : BasicSetupHelper
{
[Test]
[Category("Targeted")]
public void TestReadEmptyBlocklistInIndexFile()
{
var testopts = TestOptions.Expand(new { no_encryption = true });
var opts = new Library.Main.Options(testopts);
using (var wr = new IndexVolumeWriter(opts))
{
// Write an empty blocklist (0 bytes) - should not be rejected
using (var hasher = System.Security.Cryptography.SHA256.Create())
{
var emptyHash = Convert.ToBase64String(hasher.ComputeHash(Array.Empty<byte>()));
wr.WriteBlocklist(emptyHash, Array.Empty<byte>(), 0, 0);
}
// Write a valid blocklist
using (var hasher = System.Security.Cryptography.SHA256.Create())
{
var data = new byte[32];
new Random(42).NextBytes(data);
var hash = Convert.ToBase64String(hasher.ComputeHash(data));
wr.WriteBlocklist(hash, data, 0, 32);
}
// Start a volume with no blocks
var blockfilename = VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Blocks, opts, VolumeWriterBase.GenerateGuid(), DateTime.UtcNow);
wr.StartVolume(blockfilename);
wr.FinishVolume("hash", 123);
wr.Close();
using (var rd = new IndexVolumeReader(opts.CompressionModule, wr.LocalFilename, opts, opts.BlockhashSize))
{
var blocklists = rd.BlockLists.ToList();
Assert.That(blocklists.Count, Is.EqualTo(2), "Expected two blocklists");
var bl = blocklists[0];
Assert.That(bl.Length, Is.EqualTo(0), "Expected blocklist with 32 bytes");
var hashes = bl.Blocklist.ToList();
Assert.That(hashes.Count, Is.EqualTo(0), "Expected one hash in blocklist");
bl = blocklists[1];
Assert.That(bl.Length, Is.EqualTo(32), "Expected blocklist with 32 bytes");
hashes = bl.Blocklist.ToList();
Assert.That(hashes.Count, Is.EqualTo(1), "Expected one hash in blocklist");
var volumes = rd.Volumes.ToList();
Assert.That(volumes.Count, Is.EqualTo(1), "Expected one volume");
var vol = volumes[0];
var blocks = vol.Blocks.ToList();
Assert.That(blocks.Count, Is.EqualTo(0), "Expected no blocks in volume");
}
}
}
[Test]
[Category("Targeted")]
public void TestReadIndexFileWithOnlyBlocklists()
{
var testopts = TestOptions.Expand(new { no_encryption = true });
var opts = new Library.Main.Options(testopts);
using (var wr = new IndexVolumeWriter(opts))
{
// Write a blocklist without any volume entry
using (var hasher = System.Security.Cryptography.SHA256.Create())
{
var data = new byte[32];
new Random(42).NextBytes(data);
var hash = Convert.ToBase64String(hasher.ComputeHash(data));
wr.WriteBlocklist(hash, data, 0, 32);
}
wr.Close();
using (var rd = new IndexVolumeReader(opts.CompressionModule, wr.LocalFilename, opts, opts.BlockhashSize))
{
var blocklists = rd.BlockLists.ToList();
Assert.That(blocklists.Count, Is.EqualTo(1), "Expected one blocklist");
var volumes = rd.Volumes.ToList();
Assert.That(volumes.Count, Is.EqualTo(0), "Expected no volumes");
}
}
}
[Test]
[Category("Targeted")]
public void TestReadIndexFileSkipsInvalidEntries()
{
var testopts = TestOptions.Expand(new { no_encryption = true });
var opts = new Library.Main.Options(testopts);
using (var wr = new IndexVolumeWriter(opts))
{
// Write a valid blocklist
using (var hasher = System.Security.Cryptography.SHA256.Create())
{
var data = new byte[32];
new Random(42).NextBytes(data);
var hash = Convert.ToBase64String(hasher.ComputeHash(data));
wr.WriteBlocklist(hash, data, 0, 32);
}
// Add a volume entry with a valid filename
var blockfilename = VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Blocks, opts, VolumeWriterBase.GenerateGuid(), DateTime.UtcNow);
wr.StartVolume(blockfilename);
wr.AddBlock("abc123", 100);
wr.FinishVolume("hash", 123);
wr.Close();
using (var rd = new IndexVolumeReader(opts.CompressionModule, wr.LocalFilename, opts, opts.BlockhashSize))
{
var blocklists = rd.BlockLists.ToList();
Assert.That(blocklists.Count, Is.EqualTo(1), "Expected one blocklist");
var volumes = rd.Volumes.ToList();
Assert.That(volumes.Count, Is.EqualTo(1), "Expected one volume");
}
}
}
[Test]
[Category("Targeted")]
public void TestReadIndexFileWithEmptyBlocksArray()
{
var testopts = TestOptions.Expand(new { no_encryption = true });
var opts = new Library.Main.Options(testopts);
using (var wr = new IndexVolumeWriter(opts))
{
// Start a volume with no blocks
var blockfilename = VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Blocks, opts, VolumeWriterBase.GenerateGuid(), DateTime.UtcNow);
wr.StartVolume(blockfilename);
wr.FinishVolume("hash", 123);
wr.Close();
using (var rd = new IndexVolumeReader(opts.CompressionModule, wr.LocalFilename, opts, opts.BlockhashSize))
{
var volumes = rd.Volumes.ToList();
Assert.That(volumes.Count, Is.EqualTo(1), "Expected one volume");
var vol = volumes[0];
var blocks = vol.Blocks.ToList();
Assert.That(blocks.Count, Is.EqualTo(0), "Expected no blocks in volume");
Assert.That(vol.Hash, Is.EqualTo("hash"));
Assert.That(vol.Length, Is.EqualTo(123));
}
}
}
[Test]
[Category("Targeted")]
public void TestFinishVolumeWithNullHashProducesValidJson()
{
var testopts = TestOptions.Expand(new { no_encryption = true });
var opts = new Library.Main.Options(testopts);
using (var wr = new IndexVolumeWriter(opts))
{
var blockfilename = VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Blocks, opts, VolumeWriterBase.GenerateGuid(), DateTime.UtcNow);
wr.StartVolume(blockfilename);
wr.FinishVolume(null, 0);
wr.Close();
using (var rd = new IndexVolumeReader(opts.CompressionModule, wr.LocalFilename, opts, opts.BlockhashSize))
{
var volumes = rd.Volumes.ToList();
Assert.That(volumes.Count, Is.EqualTo(1), "Expected one volume");
var vol = volumes[0];
var blocks = vol.Blocks.ToList();
Assert.That(blocks.Count, Is.EqualTo(0), "Expected no blocks in volume");
Assert.That(vol.Hash, Is.Null);
Assert.That(vol.Length, Is.EqualTo(0));
}
}
}
[Test]
[Category("Targeted")]
public void TestReadIndexFileWithMissingVolumeHash()
{
var testopts = TestOptions.Expand(new { no_encryption = true });
var opts = new Library.Main.Options(testopts);
using (var tmpfile = new TempFile())
{
// Manually create an index file with the old broken format: {"blocks":[]}
using (var fs = new FileStream(tmpfile, FileMode.Create, FileAccess.Write))
using (var compression = Library.DynamicLoader.CompressionLoader.GetModule(opts.CompressionModule, fs, ArchiveMode.Write, opts.RawOptions))
{
// Write manifest file required by the reader
using (var sr = new StreamWriter(compression.CreateFile("manifest", CompressionHint.Compressible, DateTime.UtcNow)))
sr.Write("{\"Version\":2,\"Encoding\":\"utf8\",\"Blocksize\":" + opts.Blocksize + ",\"Created\":\"" + DateTime.UtcNow.ToString("o") + "\",\"BlockHash\":\"" + opts.BlockHashAlgorithm + "\",\"FileHash\":\"" + opts.FileHashAlgorithm + "\",\"AppVersion\":\"test\"}");
using (var sw = new StreamWriter(compression.CreateFile("vol/" + VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Blocks, opts, VolumeWriterBase.GenerateGuid(), DateTime.UtcNow), CompressionHint.Compressible, DateTime.UtcNow)))
using (var jw = new JsonTextWriter(sw))
{
jw.WriteStartObject();
jw.WritePropertyName("blocks");
jw.WriteStartArray();
jw.WriteEndArray();
jw.WriteEndObject();
}
}
using (var rd = new IndexVolumeReader(opts.CompressionModule, tmpfile, opts, opts.BlockhashSize))
{
var volumes = rd.Volumes.ToList();
Assert.That(volumes.Count, Is.EqualTo(1), "Expected one volume");
var vol = volumes[0];
var blocks = vol.Blocks.ToList();
Assert.That(blocks.Count, Is.EqualTo(0), "Expected no blocks in volume");
Assert.That(vol.Hash, Is.Null);
Assert.That(vol.Length, Is.EqualTo(0));
}
}
}
}