// Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; using uplink.NET.Interfaces; using uplink.NET.Models; using uplink.NET.Services; namespace Duplicati.Library.Backend.Storj { public class Storj : IStreamingBackend { private const string STORJ_AUTH_METHOD = "storj-auth-method"; private const string STORJ_SATELLITE = "storj-satellite"; private const string STORJ_API_KEY = "storj-api-key"; private const string STORJ_SECRET = "storj-secret"; private const string STORJ_SHARED_ACCESS = "storj-shared-access"; private const string STORJ_BUCKET = "storj-bucket"; private const string STORJ_FOLDER = "storj-folder"; private const string PROTOCOL_KEY = "storj"; private const string STORJ_PARTNER_ID = "duplicati"; private readonly string _satellite; private readonly string _api_key; private readonly string _secret; private readonly string _bucket; private readonly string _folder; private Access _access; private IBucketService _bucketService; private IObjectService _objectService; public static readonly Dictionary KNOWN_STORJ_SATELLITES = new Dictionary(StringComparer.OrdinalIgnoreCase){ { "US Central", "us1.storj.io:7777" }, { "Asia East", "ap1.storj.io:7777" }, { "Europe", "eu1.storj.io:7777" }, }; public static readonly Dictionary KNOWN_AUTHENTICATION_METHODS = new Dictionary(StringComparer.OrdinalIgnoreCase){ { "API key", "API key" }, { "Access grant", "Access grant" }, }; [DllImport("kernel32.dll")] protected static extern IntPtr LoadLibrary(string filename); private static bool _libraryLoaded = false; private static void InitStorjLibrary() { if (_libraryLoaded) return; if (Duplicati.Library.Common.Platform.IsClientWindows) //We need to init only on Windows to distinguish between x64 and x86 { if (System.Environment.Is64BitProcess) { var res = LoadLibrary("win-x64/storj_uplink.dll"); } else { var res = LoadLibrary("win-x86/storj_uplink.dll"); } } Access.SetTempDirectory(Library.Utility.TempFolder.SystemTempPath); _libraryLoaded = true; } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. public Storj() { } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. public Storj(string url, Dictionary options) { InitStorjLibrary(); foreach(var option in options.ToList()) { if(option.Key.ToLower().Contains("tardigrade")) { options.Add(option.Key.ToLower().Replace("tardigrade", "storj"), option.Value); } } var auth_method = options[STORJ_AUTH_METHOD]; if (auth_method == "Access grant") { //Create an access from the access grant var shared_access = options[STORJ_SHARED_ACCESS]; _access = new Access(shared_access, new Config() { UserAgent = STORJ_PARTNER_ID }); } else { //Create an access for a satellite, API key and encryption passphrase _satellite = options[STORJ_SATELLITE]; if (options.ContainsKey(STORJ_API_KEY)) { _api_key = options[STORJ_API_KEY]; } if (options.ContainsKey(STORJ_SECRET)) { _secret = options[STORJ_SECRET]; } _access = new Access(_satellite, _api_key, _secret, new Config() { UserAgent = STORJ_PARTNER_ID }); } _bucketService = new BucketService(_access); _objectService = new ObjectService(_access); //If no bucket was provided use the default "duplicati"-bucket if (options.ContainsKey(STORJ_BUCKET)) { _bucket = options[STORJ_BUCKET]; } else { _bucket = "duplicati"; } if (options.ContainsKey(STORJ_FOLDER)) { _folder = options[STORJ_FOLDER]; } } public string DisplayName { get { return Strings.Storj.DisplayName; } } public string ProtocolKey => PROTOCOL_KEY; public IList SupportedCommands { get { return new List(new ICommandLineArgument[] { new CommandLineArgument(STORJ_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, "API key"), new CommandLineArgument(STORJ_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, "us1.storj.io:7777"), new CommandLineArgument(STORJ_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong), new CommandLineArgument(STORJ_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong), new CommandLineArgument(STORJ_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong), new CommandLineArgument(STORJ_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong), new CommandLineArgument(STORJ_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong), }); } } public string Description { get { return Strings.Storj.Description; } } public string[] DNSName { get { return new string[0]; } } public void CreateFolder() { //Storj DCS has no folders } public void Delete(string remotename) { var deleteTask = DeleteAsync(remotename); deleteTask.Wait(); } public async Task DeleteAsync(string remotename) { try { var bucket = await _bucketService.EnsureBucketAsync(_bucket); await _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename); } catch (Exception root) { throw new FileMissingException(root); } } public void Dispose() { if (_objectService != null) { _objectService = null; } if (_bucketService != null) { _bucketService = null; } if (_access != null) { _access.Dispose(); _access = null; } } public void Get(string remotename, string filename) { var getTask = GetAsync(remotename, filename); getTask.Wait(); } public async Task GetAsync(string remotename, string filename) { var bucket = await _bucketService.EnsureBucketAsync(_bucket); var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); await download.StartDownloadAsync(); if (download.Completed) { using (FileStream file = new FileStream(filename, FileMode.Create)) { await file.WriteAsync(download.DownloadedBytes, 0, (int)download.BytesReceived); await file.FlushAsync().ConfigureAwait(false); } } } public void Get(string remotename, Stream stream) { var getTask = GetAsync(remotename, stream); getTask.Wait(); } public async Task GetAsync(string remotename, Stream stream) { int index = 0; var bucket = await _bucketService.EnsureBucketAsync(_bucket); var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); download.DownloadOperationProgressChanged += (op) => { int newPartLength = (int)op.BytesReceived - index; byte[] newPart = new byte[newPartLength]; Array.Copy(op.DownloadedBytes, index, newPart, 0, newPartLength); stream.Write(newPart, 0, newPartLength); index = index + newPartLength; }; await download.StartDownloadAsync(); } public IEnumerable List() { var listTask = ListAsync(); listTask.Wait(); return listTask.Result; } private async Task> ListAsync() { List files = new List(); var bucket = await _bucketService.EnsureBucketAsync(_bucket); var prefix = GetBasePath(); var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix }); foreach (var obj in objects.Items) { StorjFile file = new StorjFile(obj); if (prefix != "") { file.Name = file.Name.Replace(prefix, ""); } files.Add(file); } return files; } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) await PutAsync(remotename, fs, cancelToken); } public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { var bucket = await _bucketService.EnsureBucketAsync(_bucket); CustomMetadata custom = new CustomMetadata(); custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_ACCESS, Value = DateTime.Now.ToUniversalTime().ToString("O") }); custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_MODIFICATION, Value = DateTime.Now.ToUniversalTime().ToString("O") }); var upload = await _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), stream, custom, false); await upload.StartUploadAsync(); if(upload.Failed) { throw new Exception(upload.ErrorMessage); } } public void Test() { var testTask = TestAsync(); testTask.Wait(10000); if (!testTask.Result) { throw new Exception(Strings.Storj.TestConnectionFailed); } } /// /// Test the connection by: /// - creating the bucket (if it not already exists) /// - uploading 256 random bytes to a test-file /// - downloading the file back and expecting 256 bytes /// /// true, if the test was successfull or and exception private async Task TestAsync() { string testFileName = GetBasePath() + "duplicati_test.dat"; var bucket = await _bucketService.EnsureBucketAsync(_bucket); var upload = await _objectService.UploadObjectAsync(bucket, testFileName, new UploadOptions(), GetRandomBytes(256), false); await upload.StartUploadAsync(); var download = await _objectService.DownloadObjectAsync(bucket, testFileName, new DownloadOptions(), false); await download.StartDownloadAsync(); await _objectService.DeleteObjectAsync(bucket, testFileName); if (download.Failed || download.BytesReceived != 256) { throw new Exception(download.ErrorMessage); } return true; } /// /// Gets the base path - depending on there is a folder set or not /// /// The base path within a bucket where the backup shall be placed private string GetBasePath() { if (!string.IsNullOrEmpty(_folder)) return _folder + "/"; else return ""; } /// /// Creates some random bytes with the given length - just for testing the connection /// /// The length of the bytes to create /// A byte-array with the given length private static byte[] GetRandomBytes(long length) { byte[] bytes = new byte[length]; Random rand = new Random(); rand.NextBytes(bytes); return bytes; } } }