From 252f25c66ec194d1120d523cf7419e5bb9828d7a Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Tue, 25 Mar 2025 12:09:11 -0300 Subject: [PATCH] PureFTP RecursionLimit detection and Warnings This addresses the issue described on https://github.com/duplicati/duplicati/issues/6005 If a PureFTP server is detected, a warning is issued when executing server file listing. The parameter`ignore-pureftpd-limit-issue` allows the warning to be suppressed. In the event of an effective truncation, it will throw an exception and that cannot be suppressed as incomplete listing can lead to misleading backup/restore operations. --- .../Backend/FTP/AlternateFTPBackend.cs | 1 + Duplicati/Library/Backend/FTP/FTPBackend.cs | 53 ++++++++++++++++++- Duplicati/Library/Backend/FTP/Strings.cs | 5 +- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs b/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs index 85c56d74b..26384f5ac 100644 --- a/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/AlternateFTPBackend.cs @@ -78,6 +78,7 @@ public class AlternateFTPBackend : FTP new CommandLineArgument(CONFIG_KEY_FTP_UPLOAD_DELAY, CommandLineArgument.ArgumentType.Timespan, Strings.DescriptionUploadDelayShort, Strings.DescriptionUploadDelayLong, DEFAULT_UPLOAD_DELAY_STRING), new CommandLineArgument(CONFIG_KEY_FTP_LOGTOCONSOLE, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionLogToConsoleShort, Strings.DescriptionLogToConsoleLong), new CommandLineArgument(CONFIG_KEY_FTP_LOGPRIVATEINFOTOCONSOLE, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionLogPrivateInfoToConsoleShort, Strings.DescriptionLogPrivateInfoToConsoleLong, "false"), + new CommandLineArgument(CONFIG_KEY_FTP_IGNORE_PUREFTP, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionIgnorePureFTPShort, Strings.DescriptionIgnorePureFTPLong, "false"), .. SslOptionsHelper.GetCertOnlyOptions(), .. TimeoutOptionsHelper.GetOptions(), ]; diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 35c8f6ca0..096baeacc 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -27,6 +27,7 @@ using System.Security.Authentication; using System.Text; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Logging; using Duplicati.Library.Utility; using Duplicati.Library.Utility.Options; using FluentFTP; @@ -50,6 +51,7 @@ namespace Duplicati.Library.Backend /// public class FTP : IStreamingBackend { + private static readonly string LogTag = Log.LogTagFromType(typeof(FTP)); /// /// The credentials used to authenticate with the FTP server /// @@ -67,6 +69,12 @@ namespace Duplicati.Library.Backend /// private static readonly SslProtocols DEFAULT_SSL_PROTOCOLS = SslProtocols.None; // NOTE: None means "use system default" + /// + /// Configuration key for the flag to ignore the PureFTPd limit issue, suppressing the exceptions. + /// + /// Chosen not to have ftp prefix to be agnostic between aftp and ftp + /// + protected virtual string CONFIG_KEY_FTP_IGNORE_PUREFTP => "ignore-pureftpd-limit-issue"; /// /// The configuration key for the FTP encryption mode /// @@ -177,7 +185,10 @@ namespace Duplicati.Library.Backend /// The wait time after each upload before checking the file size /// private readonly TimeSpan _uploadWaitTime; - + /// + /// Flag to ignore the PureFTPd limit issue, suppressing the exceptions. + /// + private readonly bool _IgnorePureFTPdLimitIssue; /// /// The flag to indicate if the dialog should be logged to the console /// @@ -235,6 +246,7 @@ namespace Duplicati.Library.Backend new CommandLineArgument(CONFIG_KEY_FTP_LEGACY_FTPPASSIVE, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionFTPPassiveShort, Strings.DescriptionFTPPassiveLong, "false", null, null, Strings.FtpPassiveDeprecated), new CommandLineArgument(CONFIG_KEY_FTP_LEGACY_FTPREGULAR, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionFTPActiveShort, Strings.DescriptionFTPActiveLong, "true", null, null, Strings.FtpActiveDeprecated), new CommandLineArgument(CONFIG_KEY_FTP_LEGACY_USESSL, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionUseSSLShort, Strings.DescriptionUseSSLLong, "false", null, null, Strings.UseSslDeprecated), + new CommandLineArgument(CONFIG_KEY_FTP_IGNORE_PUREFTP, CommandLineArgument.ArgumentType.Boolean, Strings.DescriptionIgnorePureFTPShort, Strings.DescriptionIgnorePureFTPLong, "false"), .. SslOptionsHelper.GetCertOnlyOptions(), .. TimeoutOptionsHelper.GetOptions(), ]; @@ -332,6 +344,8 @@ namespace Duplicati.Library.Backend Noop = true }; + _IgnorePureFTPdLimitIssue = CoreUtility.ParseBoolOption(options, CONFIG_KEY_FTP_IGNORE_PUREFTP); + if (_logPrivateInfoToConsole) _ftpConfig.LogHost = _ftpConfig.LogPassword = _ftpConfig.LogUserName = true; } @@ -347,6 +361,19 @@ namespace Duplicati.Library.Backend items = await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, ct => client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size, ct) ).ConfigureAwait(false); + + if (client.ServerType == FtpServer.PureFTPd) + { + // If the list was truncated an exception has to be raised as the listing is incomplete and can lead to misleading backup/restore results + if( client.LastReplies.Any(x => + x.Code == "226" && + x.Message.Contains("truncated", StringComparison.InvariantCultureIgnoreCase))) + throw new UserInformationException("PureFTPd server effectively truncated the listing due to LimitRecursion parameter - please check documentation for more information", "PureFTPdTruncatedListing"); + + // If no truncation occured and the ignore flag is not set, issue an advisory message + if (!_IgnorePureFTPdLimitIssue) + Log.WriteWarningMessage(LogTag, "PureFTPdIssue", null, Strings.DescriptionIgnorePureFTPLong); + } } catch (Exception e) { @@ -507,6 +534,28 @@ namespace Duplicati.Library.Backend /// public async Task TestAsync(CancellationToken cancellationToken) { + + // Start with a simple list and pureFTP detection + try + { + var client = await CreateClient(cancellationToken).ConfigureAwait(false); + await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, async ct => + { + await ListAsync(cancellationToken).AnyAsync(cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + + if (client.ServerType == FtpServer.PureFTPd && !_IgnorePureFTPdLimitIssue) + throw new UserInformationException(Strings.DescriptionIgnorePureFTPLong, "PureFTPdDetected"); + + } + catch (Exception e) + { + if (TranslateException(null, ref e)) + throw e; + + throw; + } + // Try to set the working directory to trigger a folder-not-found exception try { @@ -531,7 +580,7 @@ namespace Duplicati.Library.Backend // Remove the file if it exists try { - if (await ListAsync(cancellationToken).AnyAsync(entry => entry.Name == TEST_FILE_NAME).ConfigureAwait(false)) + if (await ListAsync(cancellationToken).AnyAsync(entry => entry.Name == TEST_FILE_NAME, cancellationToken: cancellationToken).ConfigureAwait(false)) await DeleteAsync(TEST_FILE_NAME, cancellationToken).ConfigureAwait(false); } catch (Exception e) diff --git a/Duplicati/Library/Backend/FTP/Strings.cs b/Duplicati/Library/Backend/FTP/Strings.cs index f9b4ddc67..7aa1ffc0f 100644 --- a/Duplicati/Library/Backend/FTP/Strings.cs +++ b/Duplicati/Library/Backend/FTP/Strings.cs @@ -69,6 +69,9 @@ namespace Duplicati.Library.Backend public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to interpret the url path as a path that is relative to the initial login folder. This option only has an effect if the initial starting folder in the FTP server is not the (virtual) root folder. If not set, the path in the url is treated as absolute, ignoring the initial login folder."); } } public static string DescriptionUseCwdNamesShort { get { return LC.L(@"Use CWD instead of absolute paths"); } } public static string DescriptionUseCwdNamesLong { get { return LC.L(@"Use this option to start the connection with a CWD command instead of an absolute path. This can be useful if the FTP server does not support absolute paths."); } } - public static string ErrorCreateFolder(string targetFolderName, string? resultingFolder) { return LC.L(@"Error creating folder {0}, gave folder: {1}", targetFolderName, resultingFolder); } + public static string ErrorCreateFolder(string targetFolderName, string resultingFolder) { return LC.L(@"Error creating folder {0}, gave folder: {1}", targetFolderName, resultingFolder); } + public static string DescriptionIgnorePureFTPShort => LC.L(@"Ignore PureFTPd limit warnings"); + public static string DescriptionIgnorePureFTPLong => LC.L(@"PureFTPd is known to truncate file listings. If server has been configured to a higher limit or do not expect to store more than 10000 files you can suppress errors and warnings with {0}", "--ignore-pureftpd-limit-issue"); + } }