Adds error-correction (parity) data for remote data volumes so they can be repaired after bit-rot or corruption on the backend, addressing issue #314 (and reviving the intent of the stale draft PR #4574, redesigned for the current architecture). A pluggable IParity module system is introduced, mirroring the existing compression/encryption modules (IParity + ParityLoader + a new Duplicati.Library.Parity project). The default Par2Parity implementation shells out to the "par2" (par2cmdline) program. par2 emits an index file plus recovery volume files; these are concatenated into a single parity file (the PAR2 format is a stream of self-describing packets, so this is a valid PAR2 file). All par2 operations run in a temp working directory using a fixed canonical name so create and repair stay symmetric despite differing temp file names. Behavior: - Upload (Put): after a data volume (dblock/dlist) is uploaded, a parity companion "<name>.par2" is generated from the final encrypted bytes and uploaded unencrypted and untracked, inline on the held backend. dindex is skipped (regenerable). Generated post-upload so it always keys off the settled filename (retry/rename safe). - Download (Get): on a hash/size mismatch, the parity companion is fetched and used to repair the file in place before decryption. This single choke point covers Restore/Compact/Test/Repair. - Delete: the parity companion is removed (best-effort) with its data volume. - Parsing: parity files are recognized by ParsedVolume.Parse (avoiding the greedy encryption-group trap) and excluded from remote-list reconciliation so they are never flagged as extra. Parity is opt-in via --parity-redundancy-level (>0), --parity-module (default par2), with --no-parity and par2-specific options. It is best-effort: if the par2 program is absent the backup still succeeds with a single warning, and repair falls through to the normal hash-mismatch error. Not tracked in the database and requires no schema migration; the parity name is always derivable from the data volume name. Adds NUnit tests: parity filename generation/parsing and loader discovery (no external dependency), plus end-to-end Par2Parity create/verify/repair tests that are skipped when the par2 program is not available. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
153 lines
6.3 KiB
C#
153 lines
6.3 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Parity;
|
|
|
|
namespace Duplicati.Library.DynamicLoader
|
|
{
|
|
/// <summary>
|
|
/// Loads all parity modules dynamically and exposes a list of those
|
|
/// </summary>
|
|
public class ParityLoader
|
|
{
|
|
/// <summary>
|
|
/// Implementation overrides specific to parity
|
|
/// </summary>
|
|
private class ParityLoaderSub : DynamicLoader<IParity>
|
|
{
|
|
/// <summary>
|
|
/// Returns the filename extension, which is also the key
|
|
/// </summary>
|
|
/// <param name="item">The item to load the key for</param>
|
|
/// <returns>The file extension used by the module</returns>
|
|
protected override string GetInterfaceKey(IParity item)
|
|
{
|
|
return item.FilenameExtension;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the subfolders searched for parity modules
|
|
/// </summary>
|
|
protected override string[] Subfolders => ["parity"];
|
|
|
|
/// <summary>
|
|
/// The built-in modules
|
|
/// </summary>
|
|
protected override IEnumerable<IParity> BuiltInModules => ParityModules.BuiltInParityModules;
|
|
|
|
/// <summary>
|
|
/// Instanciates a specific parity module, given the file extension and options
|
|
/// </summary>
|
|
/// <param name="fileExtension">The file extension to create the instance for</param>
|
|
/// <param name="options">The options to pass to the instance constructor</param>
|
|
/// <returns>The instanciated parity module or null if the file extension is not supported</returns>
|
|
public IParity GetModule(string fileExtension, Dictionary<string, string> options)
|
|
{
|
|
if (string.IsNullOrEmpty(fileExtension))
|
|
throw new ArgumentNullException(nameof(fileExtension));
|
|
|
|
LoadInterfaces();
|
|
|
|
lock (m_lock)
|
|
{
|
|
if (m_interfaces.ContainsKey(fileExtension))
|
|
return (IParity)Activator.CreateInstance(m_interfaces[fileExtension].GetType(), options);
|
|
else
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the supported commands for a certain key
|
|
/// </summary>
|
|
/// <param name="key">The key to find commands for</param>
|
|
/// <returns>The supported commands or null if the key was not found</returns>
|
|
public IReadOnlyList<ICommandLineArgument> GetSupportedCommands(string key)
|
|
{
|
|
if (string.IsNullOrEmpty(key))
|
|
throw new ArgumentNullException(nameof(key));
|
|
|
|
LoadInterfaces();
|
|
|
|
lock (m_lock)
|
|
{
|
|
IParity b;
|
|
if (m_interfaces.TryGetValue(key, out b) && b != null)
|
|
return GetSupportedCommandsCached(b).ToList();
|
|
else
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The static instance used to access parity module information
|
|
/// </summary>
|
|
private static readonly ParityLoaderSub _parityLoader = new ParityLoaderSub();
|
|
|
|
#region Public static API
|
|
|
|
/// <summary>
|
|
/// Gets a list of loaded parity modules, the instances can be used to extract interface information, not used to interact with the module.
|
|
/// </summary>
|
|
public static IParity[] Modules { get { return _parityLoader.Interfaces; } }
|
|
|
|
/// <summary>
|
|
/// Gets a list of keys supported
|
|
/// </summary>
|
|
public static string[] Keys { get { return _parityLoader.Keys; } }
|
|
|
|
/// <summary>
|
|
/// Gets the supported commands for a given parity module
|
|
/// </summary>
|
|
/// <param name="key">The parity module to find the commands for</param>
|
|
/// <returns>The supported commands or null if the key is not supported</returns>
|
|
public static IReadOnlyList<ICommandLineArgument> GetSupportedCommands(string key)
|
|
{
|
|
return _parityLoader.GetSupportedCommands(key);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Instanciates a specific parity module, given the file extension and options
|
|
/// </summary>
|
|
/// <param name="fileextension">The file extension to create the instance for</param>
|
|
/// <param name="options">The options to pass to the instance constructor</param>
|
|
/// <returns>The instanciated parity module or null if the file extension is not supported</returns>
|
|
public static IParity GetModule(string fileextension, Dictionary<string, string> options)
|
|
{
|
|
return _parityLoader.GetModule(fileextension, options);
|
|
}
|
|
#endregion
|
|
|
|
/// <summary>
|
|
/// Removes all parity modules except the specified ones
|
|
/// </summary>
|
|
/// <param name="allowedModules">The module keys to keep, comma-separated</param>
|
|
public static void OnlyAllowModules(string allowedModules)
|
|
=> _parityLoader.RemoveAllModulesExcept(allowedModules);
|
|
|
|
}
|
|
}
|