From 27dc64121b613097d6f6d0f21b4be8cb2a13e7ef Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 16 Jan 2026 11:09:13 +0100 Subject: [PATCH] Implemented cross-tenant email restore --- Duplicati/CommandLine/CLI/Commands.cs | 8 +- .../RestoreDestinationProviderLoader.cs | 9 +- .../Interface/IRestoreDestinationProvider.cs | 12 + Duplicati/Library/Main/Controller.cs | 16 +- .../Library/Main/Operation/RestoreHandler.cs | 28 +- Duplicati/Library/Main/OperationPhase.cs | 1 + .../Builtin/FileRestoreDestinationProvider.cs | 24 +- .../RestoreDestinationProviderModules.cs | 2 +- proprietary/Office365/DTOs.cs | 74 +++ .../RestoreProvider/RestoreProvider.Email.cs | 100 ++++ .../RestoreProvider/RestoreProvider.User.cs | 409 ++++++++++++++ .../RestoreProvider/RestoreProvider.cs | 500 ++++++++++++++++++ .../Office365/SourceItems/SourceItemType.cs | 7 + .../SourceItems/UserTypeSourceEntry.cs | 35 ++ .../SourceProvider/SourceProvider.User.cs | 20 + .../SourceProvider/SourceProvider.cs | 61 ++- proprietary/Office365/Strings.cs | 7 +- proprietary/Office365/WebModule/WebModule.cs | 34 +- 18 files changed, 1280 insertions(+), 67 deletions(-) create mode 100644 proprietary/Office365/RestoreProvider/RestoreProvider.Email.cs create mode 100644 proprietary/Office365/RestoreProvider/RestoreProvider.User.cs create mode 100644 proprietary/Office365/RestoreProvider/RestoreProvider.cs diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index c5905d2d0..1610ca81a 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -723,7 +723,13 @@ namespace Duplicati.CommandLine string restorePath; options.TryGetValue("restore-path", out restorePath); - output.MessageEvent(string.Format("Restored {0} ({1}) files to {2}", res.RestoredFiles, Library.Utility.Utility.FormatSizeString(res.SizeOfRestoredFiles), string.IsNullOrEmpty(restorePath) ? "original path" : restorePath)); + if (string.IsNullOrWhiteSpace(restorePath)) + restorePath = "original location"; + if (restorePath.StartsWith("@")) + restorePath = "remote destination"; + + + output.MessageEvent(string.Format("Restored {0} ({1}) files to {2}", res.RestoredFiles, Library.Utility.Utility.FormatSizeString(res.SizeOfRestoredFiles), restorePath)); output.MessageEvent(string.Format("Duration of restore: {0:hh\\:mm\\:ss}", res.Duration)); if (output.FullResults) diff --git a/Duplicati/Library/DynamicLoader/RestoreDestinationProviderLoader.cs b/Duplicati/Library/DynamicLoader/RestoreDestinationProviderLoader.cs index fbed246fc..1d7e1786a 100644 --- a/Duplicati/Library/DynamicLoader/RestoreDestinationProviderLoader.cs +++ b/Duplicati/Library/DynamicLoader/RestoreDestinationProviderLoader.cs @@ -67,7 +67,7 @@ namespace Duplicati.Library.DynamicLoader /// The url to create the instance for /// The options to pass to the instance constructor /// The instanciated SourceProvider or null if the url is not supported - public IRestoreDestinationProvider GetRestoreDestinationProvider(string url, string mountPoint, Dictionary options) + public IRestoreDestinationProvider GetRestoreDestinationProvider(string url, Dictionary options) { var uri = new Utility.Uri(url); @@ -82,7 +82,7 @@ namespace Duplicati.Library.DynamicLoader try { if (m_interfaces.ContainsKey(uri.Scheme)) - return (IRestoreDestinationProvider)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, mountPoint, newOpts); + return (IRestoreDestinationProvider)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, newOpts); } catch (System.Reflection.TargetInvocationException tex) { @@ -172,14 +172,13 @@ namespace Duplicati.Library.DynamicLoader /// Instanciates a specific SourceProvider, given the url and options /// /// The url to create the instance for - /// The mount point to use /// The options to pass to the instance constructor /// The cancellation token /// The instanciated SourceProvider or null if the url is not supported - public static async Task GetRestoreDestinationProvider(string url, string mountPoint, Dictionary options, CancellationToken cancellationToken) + public static async Task GetRestoreDestinationProvider(string url, Dictionary options, CancellationToken cancellationToken) { // Source providers are preferred over backends - var provider = _RestoreDestinationProvider.GetRestoreDestinationProvider(url, mountPoint, options); + var provider = _RestoreDestinationProvider.GetRestoreDestinationProvider(url, options); // TODO: Support restoring to backends as well // if (provider == null) diff --git a/Duplicati/Library/Interface/IRestoreDestinationProvider.cs b/Duplicati/Library/Interface/IRestoreDestinationProvider.cs index 1ec4d6993..9dfec7e59 100644 --- a/Duplicati/Library/Interface/IRestoreDestinationProvider.cs +++ b/Duplicati/Library/Interface/IRestoreDestinationProvider.cs @@ -30,6 +30,11 @@ namespace Duplicati.Library.Interface; public interface IRestoreDestinationProvider : IDisposable { + /// + /// The target destination path + /// + string TargetDestination { get; } + /// /// Initializes the restore destination provider /// @@ -37,6 +42,13 @@ public interface IRestoreDestinationProvider : IDisposable /// An awaitable task Task Initialize(CancellationToken cancel); + /// + /// Finalizes the restore destination provider + /// + /// The cancellation token + /// An awaitable task + Task Finalize(CancellationToken cancel); + /// /// Creates the folder at the given path if it does not exist /// diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index e61b032c9..785e0a936 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -145,21 +145,27 @@ namespace Duplicati.Library.Main using var restoreDestination = (m_options.Restorepath ?? "").StartsWith("@") + // Remote destination ? await DynamicLoader.RestoreDestinationProviderLoader.GetRestoreDestinationProvider( - m_options.Restorepath, - "", // TODO: Do we need the mount point? + m_options.Restorepath.Substring(1), m_options.RawOptions, result.TaskControl.ProgressToken) .ConfigureAwait(false) - : new SourceProvider.FileRestoreDestinationProvider( - m_options.Restorepath ?? "" // TODO: Do we need the mount point? - ); + // Local destination + : new SourceProvider.FileRestoreDestinationProvider(m_options.Restorepath ?? ""); + + if (restoreDestination == null) + throw new UserInformationException($"Could not find restore destination for path: {m_options.Restorepath}", "InvalidRestoreDestination"); await new Operation.RestoreHandler(m_options, result) .RunAsync(paths, backendManager, filter, restoreDestination) .ConfigureAwait(false); + await restoreDestination.Finalize(result.TaskControl.ProgressToken).ConfigureAwait(false); + result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); + result.EndTime = DateTime.UtcNow; + UsageReporter.Reporter.Report("RESTORE_FILECOUNT", result.RestoredFiles); UsageReporter.Reporter.Report("RESTORE_FILESIZE", result.SizeOfRestoredFiles); UsageReporter.Reporter.Report("RESTORE_DURATION", (long)result.Duration.TotalSeconds); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index e8b4f5211..5f7c1e213 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -334,11 +334,11 @@ namespace Duplicati.Library.Main.Operation // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - await PrepareBlockAndFileList(database, m_options, filter, m_result) + await PrepareBlockAndFileList(database, m_options, filter, restoreDestination, m_result) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - await CreateDirectoryStructure(database, restoreDestination, string.IsNullOrEmpty(m_options.Restorepath), m_options, m_result).ConfigureAwait(false); + await CreateDirectoryStructure(database, restoreDestination, string.IsNullOrEmpty(restoreDestination.TargetDestination), m_options, m_result).ConfigureAwait(false); // At this point, there should be no more writes to the database, so we have to unlock the database: await database.Transaction @@ -433,7 +433,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Finalize); m_result.EndTime = DateTime.UtcNow; } @@ -467,13 +467,13 @@ namespace Duplicati.Library.Main.Operation //Figure out what files are to be patched, and what blocks are needed m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - await PrepareBlockAndFileList(database, m_options, filter, m_result) + await PrepareBlockAndFileList(database, m_options, filter, restoreDestination, m_result) .ConfigureAwait(false); //Make the entire output setup m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - await CreateDirectoryStructure(database, restoreDestination, string.IsNullOrEmpty(m_options.Restorepath), m_options, m_result) + await CreateDirectoryStructure(database, restoreDestination, string.IsNullOrEmpty(restoreDestination.TargetDestination), m_options, m_result) .ConfigureAwait(false); //If we are patching an existing target folder, do not touch stuff that is already updated @@ -484,7 +484,7 @@ namespace Duplicati.Library.Main.Operation await ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, restoreDestination, m_result).ConfigureAwait(false); //Look for existing blocks in the original source files only - if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(m_options.Restorepath)) + if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(restoreDestination.TargetDestination)) { using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) using (new Logging.Timer(LOGTAG, "ScanForExistingSourceBlocksFast", "ScanForExistingSourceBlocksFast")) @@ -653,7 +653,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Finalize); m_result.EndTime = DateTime.UtcNow; } @@ -886,7 +886,7 @@ namespace Duplicati.Library.Main.Operation await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } - private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) + private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, RestoreResults result) { // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id // Delete all entries from the temp table that are excluded by the filter(s) @@ -899,18 +899,18 @@ namespace Duplicati.Library.Main.Operation } using (new Logging.Timer(LOGTAG, "SetTargetPaths", "SetTargetPaths")) - if (!string.IsNullOrEmpty(options.Restorepath)) + if (!string.IsNullOrEmpty(restoreDestination.TargetDestination)) { // Find the largest common prefix var largest_prefix = options.DontCompressRestorePaths ? "" : await database.GetLargestPrefix(result.TaskControl.ProgressToken).ConfigureAwait(false); - Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); + Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(restoreDestination.TargetDestination)); // Set the target paths, special care with C:\ and / await database - .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath), result.TaskControl.ProgressToken) + .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(restoreDestination.TargetDestination), result.TaskControl.ProgressToken) .ConfigureAwait(false); } else @@ -937,11 +937,11 @@ namespace Duplicati.Library.Main.Operation // This part is not protected by try/catch as we need the target folder to exist if (options.Dryrun) { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", options.Restorepath); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", restoreDestination.TargetDestination); } - else if (await restoreDestination.CreateFolderIfNotExists(options.Restorepath, result.TaskControl.ProgressToken).ConfigureAwait(false)) + else if (await restoreDestination.CreateFolderIfNotExists(restoreDestination.TargetDestination, result.TaskControl.ProgressToken).ConfigureAwait(false)) { - Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Created root restore folder: {0}", options.Restorepath); + Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Created root restore folder: {0}", restoreDestination.TargetDestination); result.RestoredFolders++; } diff --git a/Duplicati/Library/Main/OperationPhase.cs b/Duplicati/Library/Main/OperationPhase.cs index 4329fa65f..db09d2366 100644 --- a/Duplicati/Library/Main/OperationPhase.cs +++ b/Duplicati/Library/Main/OperationPhase.cs @@ -48,6 +48,7 @@ namespace Duplicati.Library.Main Restore_PatchWithLocalBlocks, Restore_DownloadingRemoteFiles, Restore_PostRestoreVerify, + Restore_Finalize, Restore_Complete, Recreate_Running, diff --git a/Duplicati/Library/SourceProvider/Builtin/FileRestoreDestinationProvider.cs b/Duplicati/Library/SourceProvider/Builtin/FileRestoreDestinationProvider.cs index db769b095..b4ca52d40 100644 --- a/Duplicati/Library/SourceProvider/Builtin/FileRestoreDestinationProvider.cs +++ b/Duplicati/Library/SourceProvider/Builtin/FileRestoreDestinationProvider.cs @@ -30,6 +30,10 @@ public class FileRestoreDestinationProvider(string mountedPath) : IRestoreDestin private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private static readonly string DIRSEP = Path.DirectorySeparatorChar.ToString(); + /// + public string TargetDestination => mountedPath; + + /// public Task ClearReadOnlyAttribute(string path, CancellationToken cancel) { var currentAttr = SystemIO.IO_OS.GetFileAttributes(path); @@ -47,48 +51,62 @@ public class FileRestoreDestinationProvider(string mountedPath) : IRestoreDestin return Task.FromResult(true); } + /// public Task DeleteFile(string path, CancellationToken cancel) { SystemIO.IO_OS.FileDelete(path); return Task.CompletedTask; } + /// public Task DeleteFolder(string path, CancellationToken cancel) { SystemIO.IO_OS.DirectoryDelete(path, true); return Task.CompletedTask; } + /// public void Dispose() { } + /// public Task FileExists(string path, CancellationToken cancel) => Task.FromResult(SystemIO.IO_OS.FileExists(path)); + /// public Task GetFileLength(string path, CancellationToken cancel) => Task.FromResult(SystemIO.IO_OS.FileLength(path)); + /// public Task HasReadOnlyAttribute(string path, CancellationToken cancel) { var currentAttr = SystemIO.IO_OS.GetFileAttributes(path); return Task.FromResult(currentAttr.HasFlag(FileAttributes.ReadOnly)); } + /// public Task Initialize(CancellationToken cancel) - { - return Task.CompletedTask; - } + => Task.CompletedTask; + /// + public Task Finalize(CancellationToken cancel) + => Task.CompletedTask; + + /// public Task OpenRead(string path, CancellationToken cancel) => Task.FromResult(SystemIO.IO_OS.FileOpenRead(path)); + /// + /// public Task OpenReadWrite(string path, CancellationToken cancel) => Task.FromResult(SystemIO.IO_OS.FileOpenReadWrite(path)); + /// public Task OpenWrite(string path, CancellationToken cancel) => Task.FromResult(SystemIO.IO_OS.FileOpenWrite(path)); + /// public Task WriteMetadata(string path, Dictionary metadata, bool restoreSymlinkMetadata, bool restorePermissions, CancellationToken cancel) { var wrote_something = false; diff --git a/proprietary/LoaderHelper/RestoreDestinationProviderModules.cs b/proprietary/LoaderHelper/RestoreDestinationProviderModules.cs index 8db762c77..18d1947a1 100644 --- a/proprietary/LoaderHelper/RestoreDestinationProviderModules.cs +++ b/proprietary/LoaderHelper/RestoreDestinationProviderModules.cs @@ -34,7 +34,7 @@ public static class RestoreDestinationProviderModules /// private static readonly Lazy> LicensedRestoreDestinationProvidersLazy = new(() => new IRestoreDestinationProviderModule?[] { - //LicenseHelper.HasOffice365Feature ? new Office365.RestoreProvider() : null + LicenseHelper.HasOffice365Feature ? new Office365.RestoreProvider() : null } .WhereNotNull() .ToList() diff --git a/proprietary/Office365/DTOs.cs b/proprietary/Office365/DTOs.cs index 634810cbc..148253996 100644 --- a/proprietary/Office365/DTOs.cs +++ b/proprietary/Office365/DTOs.cs @@ -1016,4 +1016,78 @@ public sealed class GraphChatMessageIdentity [JsonPropertyName("userIdentityType")] public string? UserIdentityType { get; set; } +} + +internal sealed class GraphCreatedMessage +{ + [JsonPropertyName("id")] + public string Id { get; set; } = ""; +} + +internal sealed class GraphMoveRequest +{ + [JsonPropertyName("destinationId")] + public string DestinationId { get; set; } = ""; +} + +internal sealed class GraphCreateMailFolderRequest +{ + [JsonPropertyName("displayName")] + public string DisplayName { get; set; } = ""; +} + +internal sealed class GraphEmailMessageMetadata +{ + // Only fields we want to round-trip for restore + [JsonPropertyName("isRead")] + public bool? IsRead { get; set; } + + [JsonPropertyName("importance")] + public string? Importance { get; set; } // "low" | "normal" | "high" + + [JsonPropertyName("categories")] + public List? Categories { get; set; } + + [JsonPropertyName("flag")] + public GraphFollowupFlag? Flag { get; set; } +} + +internal sealed class GraphEmailMessagePatch +{ + [JsonPropertyName("isRead")] + public bool? IsRead { get; set; } + + [JsonPropertyName("importance")] + public string? Importance { get; set; } + + [JsonPropertyName("categories")] + public List? Categories { get; set; } + + [JsonPropertyName("flag")] + public GraphFollowupFlag? Flag { get; set; } +} + +internal sealed class GraphFollowupFlag +{ + [JsonPropertyName("flagStatus")] + public string? FlagStatus { get; set; } // "notFlagged" | "flagged" | "complete" + + // Optional fields if you later include them: + [JsonPropertyName("startDateTime")] + public GraphDateTimeTimeZone? StartDateTime { get; set; } + + [JsonPropertyName("dueDateTime")] + public GraphDateTimeTimeZone? DueDateTime { get; set; } + + [JsonPropertyName("completedDateTime")] + public GraphDateTimeTimeZone? CompletedDateTime { get; set; } +} + +internal sealed class GraphDateTimeTimeZone +{ + [JsonPropertyName("dateTime")] + public string? DateTime { get; set; } // Graph commonly uses string for this object + + [JsonPropertyName("timeZone")] + public string? TimeZone { get; set; } } \ No newline at end of file diff --git a/proprietary/Office365/RestoreProvider/RestoreProvider.Email.cs b/proprietary/Office365/RestoreProvider/RestoreProvider.Email.cs new file mode 100644 index 000000000..c591595e5 --- /dev/null +++ b/proprietary/Office365/RestoreProvider/RestoreProvider.Email.cs @@ -0,0 +1,100 @@ +// Copyright (c) 2026 Duplicati Inc. All rights reserved. + +using Duplicati.Library.Logging; +using Duplicati.Proprietary.Office365.SourceItems; + +namespace Duplicati.Proprietary.Office365; + +partial class RestoreProvider +{ + private EmailRestoreHelper? _emailRestoreHelper = null; + internal EmailRestoreHelper EmailRestore => _emailRestoreHelper ??= new EmailRestoreHelper(this); + internal class EmailRestoreHelper(RestoreProvider Provider) + { + private string? _targetUserId = null; + private string? _targetMailboxId = null; + private bool _hasLoadedTargetInfo = false; + + public async Task<(string? UserId, string? MailboxId)> GetUserIdAndMailboxTarget(CancellationToken cancel) + { + if (_hasLoadedTargetInfo) + { + if (string.IsNullOrWhiteSpace(_targetUserId) || string.IsNullOrWhiteSpace(_targetMailboxId)) + return (null, null); + return (_targetUserId!, _targetMailboxId!); + } + + var target = Provider.RestoreTarget; + if (target == null) + throw new InvalidOperationException("Restore target is not set"); + + if (target.Type == SourceItemType.User) + { + _targetUserId = target.Metadata["o365:Id"]!; + _targetMailboxId = await GetDefaultRestoreTargetMailbox(_targetUserId, cancel); + } + else if (target.Type == SourceItemType.UserMailbox) + { + _targetUserId = target.Path.TrimStart(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar).Skip(1).FirstOrDefault(); + if (string.IsNullOrWhiteSpace(_targetUserId)) + throw new InvalidOperationException("User ID is not set"); + _targetMailboxId = await GetDefaultRestoreTargetMailbox(_targetUserId, cancel); + } + else if (target.Type == SourceItemType.UserMailboxFolder) + { + _targetUserId = target.Path.TrimStart(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar).Skip(1).FirstOrDefault(); + _targetMailboxId = target.Metadata["o365:Id"]; + } + else + { + Log.WriteWarningMessage(LOGTAG, "RestoreUserEmailsInvalidTargetType", null, $"Restore target type {target.Type} is not valid for restoring emails."); + } + + // Don't try to load again as that could lead to repeated warnings being logged + _hasLoadedTargetInfo = true; + + if (string.IsNullOrWhiteSpace(_targetUserId) || string.IsNullOrWhiteSpace(_targetMailboxId)) + { + Log.WriteWarningMessage(LOGTAG, "RestoreUserEmailsMissingIds", null, $"Missing target userId or mailboxId for restoring emails."); + return (null, null); + } + + return (_targetUserId, _targetMailboxId); + } + + private async Task GetDefaultRestoreTargetMailbox(string userId, CancellationToken cancel) + { + const string RESTORED_FOLDER_NAME = "Restored"; + + var targetNames = new[] { RESTORED_FOLDER_NAME }; + + var path = string.Join("/", new[] { + Uri.EscapeDataString(Office365MetaType.Users.ToString().ToLowerInvariant()), + Uri.EscapeDataString(userId), + Uri.EscapeDataString(Office365UserType.Mailbox.ToString().ToLowerInvariant()) + }); + + var mailbox = await Provider.SourceProvider.GetEntry(path, true, cancel); + if (mailbox == null) + throw new InvalidOperationException($"Mailbox not found for user {userId}"); + + await foreach (var folder in mailbox.Enumerate(cancel)) + { + if (cancel.IsCancellationRequested) + break; + + var folderMetadata = await folder.GetMinorMetadata(cancel); + var folderId = folderMetadata["o365:Id"]; + if (string.IsNullOrWhiteSpace(folderId)) + continue; + + var folderName = folderMetadata["o365:Name"]; + if (targetNames.Contains(folderName, StringComparer.OrdinalIgnoreCase)) + return folderId; + } + + var restoredFolder = await Provider.EmailApi.CreateMailFolderAsync(userId, "msgfolderroot", RESTORED_FOLDER_NAME, cancel); + return restoredFolder.Id; + } + } +} diff --git a/proprietary/Office365/RestoreProvider/RestoreProvider.User.cs b/proprietary/Office365/RestoreProvider/RestoreProvider.User.cs new file mode 100644 index 000000000..c0a9f70f9 --- /dev/null +++ b/proprietary/Office365/RestoreProvider/RestoreProvider.User.cs @@ -0,0 +1,409 @@ +// Copyright (c) 2026 Duplicati Inc. All rights reserved. + +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Duplicati.Proprietary.Office365; + +partial class RestoreProvider +{ + internal EmailApiImpl EmailApi => new EmailApiImpl(_apiHelper); + + internal class EmailApiImpl(APIHelper provider) + { + public async Task RestoreEmailToFolderAsync( + string userId, + string targetFolderId, + Stream contentStream, + Stream metadataStream, + CancellationToken ct) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var user = System.Uri.EscapeDataString(userId); + + // 1) Create draft message via MIME (always goes to Drafts by default) + // POST /users/{id}/messages (MIME = base64 in text/plain body) + var createUrl = $"{baseUrl}/v1.0/users/{user}/messages"; + + using var base64TempFile = new Library.Utility.TempFile(); + + // Produce base64(MIME) as a stream without holding it in memory. + // We write once to disk so retries can reopen a fresh stream. + if (contentStream.CanSeek) contentStream.Position = 0; + + await using (var outFs = new FileStream(base64TempFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) + await using (var crypto = new System.Security.Cryptography.CryptoStream( + outFs, + new System.Security.Cryptography.ToBase64Transform(), + System.Security.Cryptography.CryptoStreamMode.Write)) + { + await contentStream.CopyToAsync(crypto, ct).ConfigureAwait(false); + crypto.FlushFinalBlock(); + await outFs.FlushAsync(ct).ConfigureAwait(false); + } + + async Task createRequestFactory(CancellationToken rct) + { + var req = new HttpRequestMessage(HttpMethod.Post, createUrl); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, rct).ConfigureAwait(false); + + req.Content = new StreamContent(new FileStream(base64TempFile, FileMode.Open, FileAccess.Read, FileShare.Read)); + req.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain"); + return req; + } + + using var createResp = await provider.SendWithRetryAsync( + createRequestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + ct).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(createResp, ct).ConfigureAwait(false); + + var createdId = await ReadIdAsync(createResp, ct).ConfigureAwait(false); + + // 2) Move to target folder (move returns a NEW message id) + // POST /users/{id}/messages/{id}/move + var moveUrl = $"{baseUrl}/v1.0/users/{user}/messages/{Uri.EscapeDataString(createdId)}/move"; + + async Task moveRequestFactory(CancellationToken rct) + { + var req = new HttpRequestMessage(HttpMethod.Post, moveUrl); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, rct).ConfigureAwait(false); + + var moveBody = JsonSerializer.SerializeToUtf8Bytes(new GraphMoveRequest { DestinationId = targetFolderId }); + req.Content = new ByteArrayContent(moveBody); + req.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + return req; + } + + using var moveResp = await provider.SendWithRetryAsync( + moveRequestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + ct).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(moveResp, ct).ConfigureAwait(false); + + var movedId = await ReadIdAsync(moveResp, ct).ConfigureAwait(false); + + if (metadataStream != null) + { + // 3) Patch metadata on moved message + await PatchRestoredEmailMetadataAsync( + userId, + movedId, + metadataStream, + ct).ConfigureAwait(false); + } + + return movedId; + } + + private static async Task ReadIdAsync(HttpResponseMessage resp, CancellationToken ct) + { + await using var s = await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + var created = await JsonSerializer.DeserializeAsync(s, cancellationToken: ct) + .ConfigureAwait(false); + + if (created?.Id is null || created.Id.Length == 0) + throw new InvalidOperationException("Graph did not return an id in the response body."); + + return created.Id; + } + + public async Task PatchRestoredEmailMetadataAsync( + string userIdOrUpn, + string restoredMessageId, + Stream metadataStream, // seekable + CancellationToken cancellationToken) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var user = Uri.EscapeDataString(userIdOrUpn); + var msg = Uri.EscapeDataString(restoredMessageId); + + var url = $"{baseUrl}/v1.0/users/{user}/messages/{msg}"; + + var jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + // Deserialize from the seekable stream (no buffering) + if (metadataStream.CanSeek) metadataStream.Position = 0; + var metadata = await JsonSerializer.DeserializeAsync( + metadataStream, jsonOptions, cancellationToken).ConfigureAwait(false) + ?? new GraphEmailMessageMetadata(); + + var patch = new GraphEmailMessagePatch + { + IsRead = metadata.IsRead, + Importance = metadata.Importance, + Categories = metadata.Categories, + Flag = metadata.Flag + }; + + // Write PATCH payload to a temp file so retries can reopen a fresh stream + var tempPath = Path.Combine(Path.GetTempPath(), $"msgpatch_{Guid.NewGuid():N}.json"); + try + { + await using (var fs = new FileStream(tempPath, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + { + await JsonSerializer.SerializeAsync(fs, patch, jsonOptions, cancellationToken).ConfigureAwait(false); + await fs.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + async Task requestFactory(CancellationToken ct) + { + var req = new HttpRequestMessage(HttpMethod.Patch, new Uri(url)); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, ct).ConfigureAwait(false); + + // New stream per attempt (safe for retries; avoids in-memory buffering) + var bodyStream = new FileStream(tempPath, FileMode.Open, FileAccess.Read, FileShare.Read); + req.Content = new StreamContent(bodyStream); + req.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + return req; + } + + using var resp = await provider.SendWithRetryAsync( + requestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + cancellationToken).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(resp, cancellationToken).ConfigureAwait(false); + } + finally + { + try { File.Delete(tempPath); } catch { /* ignore */ } + } + } + + public async Task GetChildFolderAsync( + string userIdOrUpn, + string parentFolderId, + string displayName, + CancellationToken ct) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var user = Uri.EscapeDataString(userIdOrUpn); + var parent = Uri.EscapeDataString(parentFolderId); + var filterName = displayName.Replace("'", "''"); + + var url = $"{baseUrl}/v1.0/users/{user}/mailFolders/{parent}/childFolders?$filter=displayName eq '{filterName}'&$top=1"; + + async Task requestFactory(CancellationToken rct) + { + var req = new HttpRequestMessage(HttpMethod.Get, new Uri(url)); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, rct).ConfigureAwait(false); + return req; + } + + using var resp = await provider.SendWithRetryAsync( + requestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + ct).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(resp, ct).ConfigureAwait(false); + + await using var stream = await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + var result = await JsonSerializer.DeserializeAsync>(stream, cancellationToken: ct).ConfigureAwait(false); + + return result?.Value?.FirstOrDefault(); + } + + public async Task CreateMailFolderAsync( + string userIdOrUpn, + string parentFolderId, + string displayName, + CancellationToken ct) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var user = Uri.EscapeDataString(userIdOrUpn); + var parent = Uri.EscapeDataString(parentFolderId); + + // POST /users/{id}/mailFolders/{id}/childFolders + var url = $"{baseUrl}/v1.0/users/{user}/mailFolders/{parent}/childFolders"; + + var body = new GraphCreateMailFolderRequest + { + DisplayName = displayName + }; + + async Task requestFactory(CancellationToken rct) + => new HttpRequestMessage(HttpMethod.Post, new Uri(url)) + { + Headers = + { + Authorization = await provider.GetAuthenticationHeaderAsync(false, rct).ConfigureAwait(false) + }, + Content = JsonContent.Create(body) + }; + + using var resp = await provider.SendWithRetryAsync( + requestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + ct).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(resp, ct).ConfigureAwait(false); + + await using var respStream = + await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + + var created = await JsonSerializer.DeserializeAsync( + respStream, + cancellationToken: ct).ConfigureAwait(false); + + if (created is null || string.IsNullOrWhiteSpace(created.Id)) + throw new InvalidOperationException("Graph did not return the created folder id."); + + return created; + } + + public async Task EmailExistsInFolderByInternetMessageIdAsync( + string userIdOrUpn, + string folderIdOrWellKnownName, + string internetMessageId, + CancellationToken ct) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var user = Uri.EscapeDataString(userIdOrUpn); + + // Escape OData string literals (single quotes doubled) + var imid = internetMessageId.Replace("'", "''"); + var folder = folderIdOrWellKnownName.Replace("'", "''"); + + // Constrain by both internetMessageId AND parentFolderId + var url = + $"{baseUrl}/v1.0/users/{user}/messages" + + $"?$filter=internetMessageId eq '{imid}' and parentFolderId eq '{folder}'" + + $"&$select=id" + + $"&$top=1"; + + using var stream = await provider.GetGraphAsStreamAsync( + url, "application/json", ct).ConfigureAwait(false); + + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: ct) + .ConfigureAwait(false); + + return doc.RootElement.TryGetProperty("value", out var value) && + value.ValueKind == JsonValueKind.Array && + value.GetArrayLength() > 0; + } + + public async Task RestoreCalendarEventToCalendarAsync( + string userId, + string targetCalendarId, + Stream eventJsonStream, + CancellationToken cancellationToken) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var user = Uri.EscapeDataString(userId); + var calendar = Uri.EscapeDataString(targetCalendarId); + + // Create event in a specific calendar: + // POST /users/{id}/calendars/{id}/events (application/json event body) + var url = $"{baseUrl}/v1.0/users/{user}/calendars/{calendar}/events"; + + // Buffer the body so retries can resend it + byte[] payload; + using (var ms = new MemoryStream()) + { + await eventJsonStream.CopyToAsync(ms, cancellationToken).ConfigureAwait(false); + payload = ms.ToArray(); + } + + async Task requestFactory(CancellationToken ct) + { + var req = new HttpRequestMessage(HttpMethod.Post, new Uri(url)); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, ct).ConfigureAwait(false); + + // Important: create new content per attempt + req.Content = new ByteArrayContent(payload); + req.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"); + + return req; + } + + using var resp = await provider.SendWithRetryAsync( + requestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + cancellationToken).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(resp, cancellationToken).ConfigureAwait(false); + } + + public async Task RestoreDriveItemToFolderAsync( + string driveId, + string targetFolderItemId, + string fileName, + Stream contentStream, + string? contentType, + CancellationToken cancellationToken) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var drive = Uri.EscapeDataString(driveId); + var folder = Uri.EscapeDataString(targetFolderItemId); + var name = Uri.EscapeDataString(fileName); + + // PUT /drives/{drive-id}/items/{parent-id}:/{filename}:/content + var url = + $"{baseUrl}/v1.0/drives/{drive}/items/{folder}:/{name}:/content"; + + async Task requestFactory(CancellationToken ct) + { + var req = new HttpRequestMessage(HttpMethod.Put, new Uri(url)); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, ct).ConfigureAwait(false); + + contentStream.Position = 0; + req.Content = new StreamContent(contentStream); + req.Content.Headers.ContentType = new MediaTypeHeaderValue(contentType ?? "application/octet-stream"); + + return req; + } + + using var resp = await provider.SendWithRetryAsync( + requestFactory, + HttpCompletionOption.ResponseHeadersRead, + null, + cancellationToken).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(resp, cancellationToken).ConfigureAwait(false); + } + + public async Task RestoreDriveItemMetadataAsync( + string driveId, + string itemId, + Stream metadataJsonStream, + CancellationToken cancellationToken) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var drive = Uri.EscapeDataString(driveId); + var item = Uri.EscapeDataString(itemId); + + var url = $"{baseUrl}/v1.0/drives/{drive}/items/{item}"; + + async Task requestFactory(CancellationToken ct) + { + var req = new HttpRequestMessage(HttpMethod.Patch, new Uri(url)); + req.Headers.Authorization = await provider.GetAuthenticationHeaderAsync(false, ct).ConfigureAwait(false); + + metadataJsonStream.Position = 0; + req.Content = new StreamContent(metadataJsonStream); + req.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + return req; + } + + using var resp = await provider.SendWithRetryAsync( + requestFactory, HttpCompletionOption.ResponseHeadersRead, null, cancellationToken).ConfigureAwait(false); + + await APIHelper.EnsureOfficeApiSuccessAsync(resp, cancellationToken).ConfigureAwait(false); + } + } +} \ No newline at end of file diff --git a/proprietary/Office365/RestoreProvider/RestoreProvider.cs b/proprietary/Office365/RestoreProvider/RestoreProvider.cs new file mode 100644 index 000000000..49f06287d --- /dev/null +++ b/proprietary/Office365/RestoreProvider/RestoreProvider.cs @@ -0,0 +1,500 @@ +// Copyright (c) 2026 Duplicati Inc. All rights reserved. + +using System.Collections.Concurrent; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; +using Duplicati.Library.Utility; +using Duplicati.Proprietary.Office365.SourceItems; + +namespace Duplicati.Proprietary.Office365; + +public partial class RestoreProvider : IRestoreDestinationProviderModule +{ + /// + /// Log tag for RestoreProvider + /// + private static readonly string LOGTAG = Log.LogTagFromType(); + + /// + /// API helper for accessing Office365 APIs + /// + private readonly APIHelper _apiHelper; + /// + /// Path to restore to + /// + private readonly string _restorePath; + /// + /// Source provider for accessing currently existing items on the destination tenant + /// + internal SourceProvider SourceProvider { get; init; } + + /// + /// Properties related to the restore target + /// + /// The path to restore to + /// The source provider entry to restore to + /// The metadata associated with the restore target + /// The type of the restore target + internal record RestoreTargetData( + string Path, + ISourceProviderEntry Entry, + Dictionary Metadata, + SourceItemType Type); + + /// + /// The restore target data + /// + internal RestoreTargetData? RestoreTarget { get; private set; } + + /// + /// Temporary files created during the restore process + /// + private readonly ConcurrentDictionary _temporaryFiles = new(); + /// + /// Metadata recorded during the restore process + /// + private readonly ConcurrentDictionary> _metadata = new(); + /// + /// Map of restored email folder paths to their corresponding IDs + /// + private readonly ConcurrentDictionary _restoredEmailFolderMap = new(); + + /// + /// Default constructor for the restore provider. + /// Only used for loading metadata about the provider. + /// + public RestoreProvider() + { + _apiHelper = null!; + _restorePath = null!; + SourceProvider = null!; + } + + /// + /// Constructs the RestoreProvider with the given URL and options. + /// + /// The destination URL for the restore operation + /// The options for the restore operation + public RestoreProvider(string url, Dictionary options) + { + var uri = new Library.Utility.Uri(url); + _restorePath = uri.HostAndPath; + + var parsedOptions = OptionsHelper.ParseAndValidateOptions(url, options); + _apiHelper = APIHelper.Create( + tenantId: parsedOptions.TenantId, + authOptions: parsedOptions.AuthOptions, + graphBaseUrl: parsedOptions.GraphBaseUrl + ); + + var sourceOpts = new Dictionary(options) + { + { "store-metadata-content-in-database", "true" } + }; + + SourceProvider = new SourceProvider("office365://", "", sourceOpts); + } + + /// + public string Key => OptionsHelper.ModuleKey; + + /// + public string DisplayName => Strings.ProviderDisplayName; + + /// + public string Description => Strings.ProviderDescription; + + /// + public IList SupportedCommands => OptionsHelper.SupportedCommands; + + /// + public string TargetDestination => _restorePath; + + /// + public Task CreateFolderIfNotExists(string path, CancellationToken cancel) + { + // TODO: Do we need to do anything here? + return Task.FromResult(false); + } + + /// + public async Task Initialize(CancellationToken cancel) + { + await _apiHelper.AcquireAccessTokenAsync(true, cancel); + await SourceProvider.Initialize(cancel); + var entry = await SourceProvider.GetEntry(_restorePath, isFolder: true, cancel); + if (entry == null) + throw new UserInformationException($"Restore target path not found: {_restorePath}", "RestoreTargetNotFound"); + var metadata = await entry.GetMinorMetadata(cancel); + var type = metadata.TryGetValue("o365:Type", out var typeStr) + && Enum.TryParse(typeStr, out var sourceItemType) + ? sourceItemType + : throw new UserInformationException($"Invalid restore target type in metadata: {typeStr}", "InvalidRestoreTargetType"); + + RestoreTarget = new RestoreTargetData(_restorePath, entry, metadata, type); + } + + /// + /// Normalizes the given path by applying the RemovedRestorePrefix if it has been removed + /// + /// The path to normalize + /// The normalized path + private string NormalizePath(string path) + => path; + + /// + public void Dispose() + { + _apiHelper?.Dispose(); + SourceProvider?.Dispose(); + + foreach (var file in _temporaryFiles.Values) + file.Dispose(); + _temporaryFiles.Clear(); + } + + /// + public async Task FileExists(string path, CancellationToken cancel) + { + path = NormalizePath(path); + + if (_temporaryFiles.ContainsKey(path)) + return true; + + var entry = await SourceProvider.GetEntry(path, isFolder: false, cancel).ConfigureAwait(false); + return entry != null; + } + + /// + public Task OpenWrite(string path, CancellationToken cancel) + { + path = NormalizePath(path); + + var file = _temporaryFiles.GetOrAdd(path, _ => new TempFile()); + return Task.FromResult(SystemIO.IO_OS.FileOpenWrite(file)); + } + + /// + public async Task OpenRead(string path, CancellationToken cancel) + { + path = NormalizePath(path); + + if (_temporaryFiles.ContainsKey(path)) + return SystemIO.IO_OS.FileOpenRead(_temporaryFiles[path]); + + var entry = await SourceProvider.GetEntry(path, isFolder: false, cancel).ConfigureAwait(false); + if (entry != null) + return await entry.OpenRead(cancel).ConfigureAwait(false); + + throw new FileNotFoundException($"File not found: {path}"); + } + + /// + public async Task OpenReadWrite(string path, CancellationToken cancel) + { + path = NormalizePath(path); + + if (_temporaryFiles.ContainsKey(path)) + return SystemIO.IO_OS.FileOpenReadWrite(_temporaryFiles[path]); + + var entry = await SourceProvider.GetEntry(path, isFolder: false, cancel).ConfigureAwait(false); + if (entry == null) + { + _temporaryFiles.GetOrAdd(path, _ => new TempFile()); + return SystemIO.IO_OS.FileOpenReadWrite(_temporaryFiles[path]); + } + + if (entry.IsFolder) + throw new FileNotFoundException($"Path is a folder: {path}"); + + var file = new TempFile(); + using (var fs = SystemIO.IO_OS.FileOpenWrite(file)) + using (var stream = await entry.OpenRead(cancel).ConfigureAwait(false)) + await stream.CopyToAsync(fs).ConfigureAwait(false); + + _temporaryFiles[path] = file; + return SystemIO.IO_OS.FileOpenReadWrite(file); + } + + /// + public async Task GetFileLength(string path, CancellationToken cancel) + { + path = NormalizePath(path); + + if (_temporaryFiles.ContainsKey(path)) + return SystemIO.IO_OS.FileLength(_temporaryFiles[path]); + + var entry = await SourceProvider.GetEntry(path, isFolder: false, cancel).ConfigureAwait(false); + if (entry == null) + throw new FileNotFoundException($"File not found: {path}"); + + return entry.Size; + } + + /// + public Task HasReadOnlyAttribute(string path, CancellationToken cancel) + => Task.FromResult(false); + + /// + public Task ClearReadOnlyAttribute(string path, CancellationToken cancel) + => Task.CompletedTask; + + /// + public Task WriteMetadata(string path, Dictionary metadata, bool restoreSymlinkMetadata, bool restorePermissions, CancellationToken cancel) + { + path = NormalizePath(path); + + _metadata.AddOrUpdate(path, metadata, (_, _) => metadata); + return Task.FromResult(true); + } + + /// + public Task DeleteFolder(string path, CancellationToken cancel) + => Task.CompletedTask; + + /// + public Task DeleteFile(string path, CancellationToken cancel) + // Should only happen if attempting to restore a symlink, which is not supported + => throw new NotImplementedException("File deletion is not supported in Office365 RestoreProvider"); + + /// + public async Task Finalize(CancellationToken cancel) + { + try + { + await RestoreUserEmailFolders(cancel).ConfigureAwait(false); + } + catch (Exception ex) + { + Log.WriteErrorMessage(LOGTAG, "FinalizeRestoreUserEmailFoldersFailed", ex, $"Failed to restore user email folders: {ex.Message}"); + } + + try + { + await RestoreUserEmails(cancel).ConfigureAwait(false); + } + catch (Exception ex) + { + Log.WriteErrorMessage(LOGTAG, "FinalizeRestoreUserEmailsFailed", ex, $"Failed to restore user emails: {ex.Message}"); + } + + await RestoreDrives(cancel).ConfigureAwait(false); + await RestoreDriveFolders(cancel).ConfigureAwait(false); + await RestoreDriveFiles(cancel).ConfigureAwait(false); + + await RestoreCalendarEvents(cancel).ConfigureAwait(false); + await RestoreContacts(cancel).ConfigureAwait(false); + + await RestoreGroupChannels(cancel).ConfigureAwait(false); + await RestoreGroupConversations(cancel).ConfigureAwait(false); + + await RestorePlannerTasks(cancel).ConfigureAwait(false); + + // We should have restored all items by now + if (_metadata.Count > 0 || _temporaryFiles.Count > 0) + Log.WriteWarningMessage(LOGTAG, "FinalizeIncomplete", null, $"Some items were not restored. Remaining metadata items: {_metadata.Count}, remaining temporary files: {_temporaryFiles.Count}"); + + var tempFiles = _temporaryFiles.Values.ToList(); + foreach (var file in tempFiles) + file.Dispose(); + _temporaryFiles.Clear(); + _metadata.Clear(); + } + + /// + /// Gets metadata entries by their source item type + /// + /// The source item type to filter by + /// The list of metadata entries matching the given type + private List>> GetMetadataByType(SourceItemType type) + => _metadata + .Where(kv => kv.Value.TryGetValue("o365:Type", out var typeStr) + && typeStr == type.ToString()) + .ToList(); + + /// + /// Restores user email folders and updates the folder ID map + /// + /// The cancellation token + /// An awaitable task + private async Task RestoreUserEmailFolders(CancellationToken cancel) + { + if (RestoreTarget == null) + throw new InvalidOperationException("Restore target entry is not set"); + + // Find email entries in _metadata + var emailFolders = GetMetadataByType(SourceItemType.UserMailboxFolder); + if (emailFolders.Count == 0) + return; + + (var userId, var mailboxId) = await EmailRestore.GetUserIdAndMailboxTarget(cancel); + if (string.IsNullOrWhiteSpace(userId) || string.IsNullOrWhiteSpace(mailboxId)) + return; // A warning has already been logged + + // Create folders + // Sort folders by path length to ensure parents are created before children + var sortedFolders = emailFolders.OrderBy(k => k.Key.Split(Path.DirectorySeparatorChar).Length).ToList(); + + foreach (var folder in sortedFolders) + { + if (cancel.IsCancellationRequested) + break; + + var originalPath = folder.Key; + var metadata = folder.Value; + var displayName = metadata.GetValueOrDefault("o365:Name") ?? metadata.GetValueOrDefault("o365:DisplayName") ?? metadata.GetValueOrDefault("o365:Id"); + + _metadata.TryRemove(originalPath, out _); + + if (string.IsNullOrWhiteSpace(displayName)) + { + Log.WriteWarningMessage(LOGTAG, "RestoreUserEmailFoldersMissingName", null, $"Missing display name for folder {originalPath}, skipping."); + continue; + } + + // Determine parent folder ID + // We assume the path structure reflects the hierarchy + // If parent path is not in map, we assume it's the root (mailboxId) + var parentPath = Util.AppendDirSeparator(Path.GetDirectoryName(originalPath.TrimEnd(Path.DirectorySeparatorChar))); + var parentId = mailboxId; + + if (parentPath != null && _restoredEmailFolderMap.TryGetValue(parentPath, out var mappedParentId)) + parentId = mappedParentId; + + try + { + // Check if folder exists + var existingFolder = await EmailApi.GetChildFolderAsync(userId, parentId, displayName, cancel); + if (existingFolder != null) + { + _restoredEmailFolderMap[originalPath] = existingFolder.Id; + } + else + { + var newFolder = await EmailApi.CreateMailFolderAsync(userId, parentId, displayName, cancel); + _restoredEmailFolderMap[originalPath] = newFolder.Id; + } + } + catch (Exception ex) + { + Log.WriteErrorMessage(LOGTAG, "RestoreUserEmailFoldersFailed", ex, $"Failed to restore folder {displayName} at {originalPath}: {ex.Message}"); + } + } + } + + /// + /// Restores user emails + /// + /// >The cancellation token + /// An awaitable task + private async Task RestoreUserEmails(CancellationToken cancel) + { + if (RestoreTarget == null) + throw new InvalidOperationException("Restore target entry is not set"); + + // Find email entries in _metadata + var emailSources = GetMetadataByType(SourceItemType.UserMailboxEmail); + if (emailSources.Count == 0) + return; + + (var userId, var mailboxId) = await EmailRestore.GetUserIdAndMailboxTarget(cancel); + if (string.IsNullOrWhiteSpace(userId) || string.IsNullOrWhiteSpace(mailboxId)) + return; // A warning has already been logged + + // Restore emails + foreach (var emailSource in emailSources) + { + if (cancel.IsCancellationRequested) + break; + + try + { + var contentPath = SystemIO.IO_OS.PathCombine(emailSource.Key, "content.eml"); + var metadataPath = SystemIO.IO_OS.PathCombine(emailSource.Key, "metadata.json"); + + var internetMessageId = emailSource.Value.GetValueOrDefault("o365:InternetMessageId"); + + // Determine target folder + var targetFolderId = mailboxId; + var parentPath = Util.AppendDirSeparator(Path.GetDirectoryName(emailSource.Key.TrimEnd(Path.DirectorySeparatorChar))); + if (parentPath != null && _restoredEmailFolderMap.TryGetValue(parentPath, out var mappedFolderId)) + { + targetFolderId = mappedFolderId; + } + + if (!string.IsNullOrWhiteSpace(internetMessageId) && await EmailApi.EmailExistsInFolderByInternetMessageIdAsync(userId, targetFolderId, internetMessageId, cancel)) + { + Log.WriteInformationMessage(LOGTAG, "RestoreUserEmailsSkipExisting", null, $"Email with InternetMessageId {internetMessageId} already exists in target mailbox, skipping restore for {emailSource.Key}."); + } + else + { + var contentEntry = _temporaryFiles.GetValueOrDefault(contentPath); + var metadataEntry = _temporaryFiles.GetValueOrDefault(metadataPath); + + if (contentEntry == null) + { + Log.WriteWarningMessage(LOGTAG, "RestoreUserEmailsMissingContent", null, $"Missing email content for {emailSource.Key}, skipping."); + continue; + } + + if (metadataEntry == null) + { + Log.WriteWarningMessage(LOGTAG, "RestoreUserEmailsMissingMetadata", null, $"Missing email metadata for {emailSource.Key}, skipping."); + continue; + } + + + using (var contentStream = SystemIO.IO_OS.FileOpenRead(contentEntry)) + using (var metadataStream = SystemIO.IO_OS.FileOpenRead(metadataEntry)) + await EmailApi.RestoreEmailToFolderAsync(userId, targetFolderId, contentStream, metadataStream, cancel); + } + + // Clean up when done + _metadata.TryRemove(emailSource.Key, out _); + _metadata.TryRemove(contentPath, out _); + _metadata.TryRemove(metadataPath, out _); + _temporaryFiles.TryRemove(contentPath, out var contentFile); + _temporaryFiles.TryRemove(metadataPath, out var metadataFile); + contentFile?.Dispose(); + metadataFile?.Dispose(); + } + catch (Exception ex) + { + Log.WriteErrorMessage(LOGTAG, "RestoreUserEmailsFailed", ex, $"Failed to restore email at {emailSource.Key}: {ex.Message}"); + } + } + } + + private async Task RestoreDrives(CancellationToken cancel) + { + } + + private async Task RestoreDriveFolders(CancellationToken cancel) + { + } + + private async Task RestoreDriveFiles(CancellationToken cancel) + { + } + + private async Task RestoreCalendarEvents(CancellationToken cancel) + { + } + + private async Task RestoreContacts(CancellationToken cancel) + { + } + + private async Task RestoreGroupChannels(CancellationToken cancel) + { + } + + private async Task RestoreGroupConversations(CancellationToken cancel) + { + } + + private async Task RestorePlannerTasks(CancellationToken cancel) + { + } +} diff --git a/proprietary/Office365/SourceItems/SourceItemType.cs b/proprietary/Office365/SourceItems/SourceItemType.cs index 22a6fd7a7..3a4919647 100644 --- a/proprietary/Office365/SourceItems/SourceItemType.cs +++ b/proprietary/Office365/SourceItems/SourceItemType.cs @@ -41,6 +41,13 @@ public enum SourceItemType UserMailboxEmail, User, UserContact, + UserProfile, + UserCalendar, + UserContacts, + UserTasks, + UserNotes, + UserChats, + UserPlannerTasks, PlannerBucket } \ No newline at end of file diff --git a/proprietary/Office365/SourceItems/UserTypeSourceEntry.cs b/proprietary/Office365/SourceItems/UserTypeSourceEntry.cs index f4c0080c1..9c29d15b0 100644 --- a/proprietary/Office365/SourceItems/UserTypeSourceEntry.cs +++ b/proprietary/Office365/SourceItems/UserTypeSourceEntry.cs @@ -12,6 +12,41 @@ internal class UserTypeSourceEntry(SourceProvider provider, string path, GraphUs { private static readonly string LOGTAG = Log.LogTagFromType(); + public override Task> GetMinorMetadata(CancellationToken cancellationToken) + => Task.FromResult(new Dictionary() + { + { "o365:v", "1" }, + { "o365:Id", user.Id }, + { "o365:Type", userType switch + { + Office365UserType.Profile => SourceItemType.UserProfile.ToString(), + Office365UserType.Mailbox => SourceItemType.UserMailbox.ToString(), + Office365UserType.Calendar => SourceItemType.UserCalendar.ToString(), + Office365UserType.Contacts => SourceItemType.UserContacts.ToString(), + Office365UserType.Tasks => SourceItemType.UserTasks.ToString(), + Office365UserType.Notes => SourceItemType.UserNotes.ToString(), + Office365UserType.Planner => SourceItemType.UserPlannerTasks.ToString(), + Office365UserType.Chats => SourceItemType.UserChats.ToString(), + _ => null + } + }, + { "o365:Name", userType switch + { + Office365UserType.Profile => "Profile", + Office365UserType.Mailbox => "Mailbox", + Office365UserType.Calendar => "Calendar", + Office365UserType.Contacts => "Contacts", + Office365UserType.Tasks => "Tasks", + Office365UserType.Notes => "Notes", + Office365UserType.Planner => "Planner", + Office365UserType.Chats => "Chats", + _ => null + } + } + } + .Where(kv => !string.IsNullOrEmpty(kv.Value)) + .ToDictionary(kv => kv.Key, kv => kv.Value)); + public override async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken cancellationToken) { var entries = userType switch diff --git a/proprietary/Office365/SourceProvider/SourceProvider.User.cs b/proprietary/Office365/SourceProvider/SourceProvider.User.cs index b88b3d006..fb89f7868 100644 --- a/proprietary/Office365/SourceProvider/SourceProvider.User.cs +++ b/proprietary/Office365/SourceProvider/SourceProvider.User.cs @@ -531,6 +531,26 @@ partial class SourceProvider var url = $"{baseUrl}/v1.0/drives/{drive}/items/{item}/content"; return provider.GetGraphAsStreamAsync(url, "application/octet-stream", ct); } + + internal Task GetDriveItemMetadataStreamAsync( + string driveId, + string itemId, + CancellationToken ct) + { + var baseUrl = provider.GraphBaseUrl.TrimEnd('/'); + var drive = Uri.EscapeDataString(driveId); + var item = Uri.EscapeDataString(itemId); + + var select = + "id,name,parentReference,webUrl,eTag,cTag,size," + + "createdDateTime,lastModifiedDateTime,createdBy,lastModifiedBy," + + "fileSystemInfo,file,folder,package,shared,deleted"; + + var url = + $"{baseUrl}/v1.0/drives/{drive}/items/{item}?$select={Uri.EscapeDataString(select)}"; + + return provider.GetGraphAsStreamAsync(url, "application/json", ct); + } } internal CalendarApiImpl CalendarApi => new CalendarApiImpl(_apiHelper); diff --git a/proprietary/Office365/SourceProvider/SourceProvider.cs b/proprietary/Office365/SourceProvider/SourceProvider.cs index 15a0885d0..4d24e88b5 100644 --- a/proprietary/Office365/SourceProvider/SourceProvider.cs +++ b/proprietary/Office365/SourceProvider/SourceProvider.cs @@ -1,5 +1,6 @@ // Copyright (c) 2026 Duplicati Inc. All rights reserved. +using System.Collections.Concurrent; using System.Runtime.CompilerServices; using Duplicati.Library.Interface; using Duplicati.Proprietary.Office365.SourceItems; @@ -13,6 +14,8 @@ public sealed partial class SourceProvider : ISourceProviderModule, IDisposable private readonly APIHelper _apiHelper; private readonly string _mountPoint; + private readonly ConcurrentDictionary _entryCache = new(); + private static readonly Office365UserType[] DELEGATED_USER_TYPES = [ Office365UserType.Tasks, @@ -47,7 +50,7 @@ public sealed partial class SourceProvider : ISourceProviderModule, IDisposable public SourceProvider(string url, string mountPoint, Dictionary options) { if (!Library.Utility.Utility.ParseBoolOption(options, "store-metadata-content-in-database")) - throw new UserInformationException(Strings.MetadataStorageNotEnabled, "DatabaseMetadataStorageNotEnabled"); + throw new UserInformationException(Strings.MetadataStorageNotEnabled("store-metadata-content-in-database"), "DatabaseMetadataStorageNotEnabled"); _mountPoint = mountPoint; var parsedOptions = OptionsHelper.ParseAndValidateOptions(url, options); @@ -82,10 +85,60 @@ public sealed partial class SourceProvider : ISourceProviderModule, IDisposable yield return new RootSourceEntry(this, _mountPoint); } - public Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) + public async Task GetEntry(string path, bool isFolder, CancellationToken cancellationToken) { - // TODO: Implement this - throw new NotImplementedException(); + if (_entryCache.TryGetValue(path, out var cachedEntry)) + return cachedEntry; + + ISourceProviderEntry resultEntry = new RootSourceEntry(this, _mountPoint); + var targetPath = path.TrimEnd(Path.DirectorySeparatorChar); + + var currentPath = targetPath; + while (!string.IsNullOrEmpty(currentPath)) + { + var parentPath = Path.GetDirectoryName(currentPath); + if (parentPath != null && _entryCache.TryGetValue(parentPath, out var cachedParent)) + { + resultEntry = cachedParent; + break; + } + currentPath = parentPath; + } + + var relativePath = string.IsNullOrEmpty(currentPath) + ? targetPath + : Path.GetRelativePath(currentPath, targetPath); + + var pathsegments = relativePath?.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty(); + if (pathsegments.Length == 0) + pathsegments = [""]; + + foreach (var item in pathsegments) + { + var found = false; + await foreach (var entry in resultEntry.Enumerate(cancellationToken)) + { + _entryCache.TryAdd(entry.Path, entry); + + var name = entry.Path.TrimEnd(Path.DirectorySeparatorChar) + .Split(Path.DirectorySeparatorChar) + .Last(); + if (name.Equals(item, StringComparison.OrdinalIgnoreCase)) + { + if (!entry.IsFolder) + throw new UserInformationException($"Path segment '{item}' is not a folder", "PathSegmentNotFolder"); + + resultEntry = entry; + found = true; + break; + } + } + + if (!found) + return null; + } + + return resultEntry; } internal IEnumerable IncludedRootTypes => diff --git a/proprietary/Office365/Strings.cs b/proprietary/Office365/Strings.cs index 5e07b0a2d..bfc71d528 100644 --- a/proprietary/Office365/Strings.cs +++ b/proprietary/Office365/Strings.cs @@ -17,11 +17,8 @@ internal static class Strings public static string MissingAccessToken => LC.L("An OAuth access token is required for this workspace backend."); public static string InvalidRestoreTargetType(string? type) => LC.L($"Invalid restore target type: {type}"); - public static string IncorrectTargetUrlFormat(RestoreTargetType targetType, int expected, string[] parts) => LC.L( - $"The target URL does not have the correct number of parts for restore target type {targetType}: expected {expected}, got {parts.Length}"); - - public static string MetadataStorageNotEnabled => LC.L( - "Storing metadata content in the database must be enabled to use the Microsoft 365 source provider."); + public static string MetadataStorageNotEnabled(string optionname) => LC.L( + $"Storing metadata content in the database must be enabled to use the Microsoft 365 source provider. Use the option: --{optionname}"); public static string MissingTenantId => LC.L("A tenant ID must be supplied to query the Office 365 Management API."); diff --git a/proprietary/Office365/WebModule/WebModule.cs b/proprietary/Office365/WebModule/WebModule.cs index 4652d0e64..186626636 100644 --- a/proprietary/Office365/WebModule/WebModule.cs +++ b/proprietary/Office365/WebModule/WebModule.cs @@ -90,38 +90,14 @@ public class WebModule : IWebModule forwardoptions[key!] = uri.QueryParameters[key]; using var client = new SourceProvider(url, "", forwardoptions); - var entries = client.Enumerate(cancellationToken); - path = path?.TrimEnd(Path.DirectorySeparatorChar); + await client.Initialize(cancellationToken); - var pathsegments = path?.Split(Path.DirectorySeparatorChar) ?? Array.Empty(); - if (pathsegments.Length == 0) - pathsegments = [""]; - - foreach (var item in pathsegments) - { - var found = false; - await foreach (var entry in entries.WithCancellation(cancellationToken)) - { - var name = entry.Path.TrimEnd(Path.DirectorySeparatorChar) - .Split(Path.DirectorySeparatorChar) - .Last(); - if (name.Equals(item, StringComparison.OrdinalIgnoreCase)) - { - if (!entry.IsFolder) - throw new UserInformationException($"Path segment '{item}' is not a folder", "PathSegmentNotFolder"); - - entries = entry.Enumerate(cancellationToken); - found = true; - break; - } - } - - if (!found) - throw new UserInformationException($"Path segment '{item}' not found", "PathSegmentNotFound"); - } + var targetEntry = await client.GetEntry(path ?? "", isFolder: true, cancellationToken).ConfigureAwait(false); + if (targetEntry == null) + throw new DirectoryNotFoundException($"Path not found: {path}"); var result = new Dictionary(); - await foreach (var entry in entries.WithCancellation(cancellationToken)) + await foreach (var entry in targetEntry.Enumerate(cancellationToken)) { if (op == Operation.ListDestinationRestoreTargets) {