// Copyright (C) 2011, Kenneth Skovhede
// http://www.hexad.dk, opensource@hexad.dk
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using Duplicati.Library.Main.Database;
using System.Collections.Generic;
using System.Linq;
namespace Duplicati.Library.Main
{
public static class FilelistProcessor
{
///
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
///
/// The backend instance to use
/// The options used
/// The database to compare with
public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, CommunicationStatistics stat)
{
var tp = RemoteListAnalysis(backend, options, database);
long extraCount = 0;
long missingCount = 0;
foreach(var n in tp.ExtraVolumes)
{
if (!options.QuietConsole)
stat.LogMessage(string.Format("Extra unknown file: {0}", n.File.Name));
stat.LogWarning(string.Format("Extra unknown file: {0}", n.File.Name), null);
extraCount++;
}
foreach(var n in tp.MissingVolumes)
{
if (!options.QuietConsole)
stat.LogMessage(string.Format("Missing file: {0}", n.Name));
stat.LogWarning(string.Format("Missing file: {0}", n.Name), null);
missingCount++;
}
if (extraCount > 0)
throw new Exception(string.Format("Found {0} remote files that are not recorded in local storage, please run cleanup", extraCount));
if (missingCount > 0)
{
if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0)
throw new Exception(string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes)));
else
throw new Exception(string.Format("Found {0} files that are missing from the remote storage, please run cleanup", missingCount));
}
}
public struct RemoteAnalysisResult
{
public IEnumerable ParsedVolumes;
public IEnumerable ExtraVolumes;
public IEnumerable MissingVolumes;
public string[] BackupPrefixes { get { return ParsedVolumes.Select(x => x.Prefix).Distinct().ToArray(); } }
}
///
/// Helper method that verifies uploaded volumes and updates their state in the database.
/// Throws an error if there are issues with the remote storage
///
/// The backend instance to use
/// The options used
/// The database to compare with
public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database)
{
var rawlist = backend.List();
var lookup = new Dictionary();
var remotelist = from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p;
foreach (var s in remotelist)
if (s.Prefix == options.Prefix)
lookup[s.File.Name] = s;
var missing = new List();
var locallist = database.GetRemoteVolumes();
foreach (var i in locallist)
{
//Ignore those that are deleted
if (i.State == RemoteVolumeState.Deleted)
continue;
if (i.State == RemoteVolumeState.Temporary)
{
database.LogMessage("info", string.Format("removing file listed as {0}: {1}", i.State, i.Name), null, null);
database.RemoveRemoteVolume(i.Name, null);
}
else
{
Volumes.IParsedVolume r;
if (!lookup.TryGetValue(i.Name, out r))
{
if (i.State == RemoteVolumeState.Uploading || i.State == RemoteVolumeState.Deleting || (r != null && r.File.Size != i.Size && r.File.Size >= 0 && i.Size >= 0))
{
database.LogMessage("info", string.Format("removing file listed as {0}: {1}", i.State, i.Name), null, null);
database.RemoveRemoteVolume(i.Name, null);
}
else
missing.Add(i);
}
else if (i.State != RemoteVolumeState.Verified)
{
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash);
}
lookup.Remove(i.Name);
}
}
return new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing };
}
internal static IEnumerable ParseFileList(string target, Dictionary options, CommunicationStatistics stat)
{
var opts = new Options(options);
using (var db = new LocalDatabase(opts.Dbpath, "ParseFileList"))
using (var b = new BackendManager(target, opts, stat, db))
{
var res =
from n in b.List()
let np = Volumes.VolumeBase.ParseFilename(n)
where np != null
select np;
b.WaitForComplete(db, null);
return res;
}
}
}
}