// Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using COSXML; using COSXML.Auth; using COSXML.Model.Bucket; using COSXML.Model.Object; using COSXML.Model.Tag; using COSXML.Utils; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Backend.TencentCOS { /// /// Tencent COS /// en: https://intl.cloud.tencent.com/document/product/436 /// zh: https://cloud.tencent.com/document/product/436 /// public class COS : IBackend, IStreamingBackend, IRenameEnabledBackend { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private const string COS_APP_ID = "cos-app-id"; private const string COS_REGION = "cos-region"; private const string COS_SECRET_ID = "cos-secret-id"; private const string COS_SECRET_KEY = "cos-secret-key"; private const string COS_BUCKET = "cos-bucket"; private const string COS_STORAGE_CLASS = "cos-storage-class"; /// /// Set default HTTPS request /// private const bool COS_USE_SSL = true; /// /// Connection timeout time, in milliseconds, default 45000ms /// private const int CONNECTION_TIMEOUT_MS = 1000 * 60 * 60; /// /// Read and write timeout time, in milliseconds, default 45000ms /// private const int READ_WRITE_TIMEOUT_MS = 1000 * 60 * 60; /// /// The valid time of each request signature, unit: second /// private const int KEY_DURATION_SECOND = 60 * 60; private readonly CosOptions _cosOptions; CosXml cosXml; /// /// Tencent Cloud Object Storage Configuration /// public class CosOptions { /// /// Tencent Cloud Account APPID /// public string Appid { get; set; } /// /// Cloud API Secret Id /// public string SecretId { get; set; } /// /// Cloud API Secret Key /// public string SecretKey { get; set; } /// /// Bucket region ap-guangzhou ap-hongkong /// en: https://intl.cloud.tencent.com/document/product/436/6224 /// zh: https://cloud.tencent.com/document/product/436/6224 /// public string Region { get; set; } /// /// Bucket, format: BucketName-APPID /// public string Bucket { get; set; } /// /// A path or subfolder in a bucket /// public string Path { get; set; } /// /// Storage class of the object /// en: https://intl.cloud.tencent.com/document/product/436/30925 /// zh: https://cloud.tencent.com/document/product/436/33417 /// public string StorageClass { get; set; } } public COS() { } public COS(string url, Dictionary options) { _cosOptions = new CosOptions(); var uri = new Utility.Uri(url?.Trim()); var prefix = uri.HostAndPath?.Trim()?.Trim('/')?.Trim('\\'); if (!string.IsNullOrEmpty(prefix)) { _cosOptions.Path = prefix + "/"; } if (options.ContainsKey(COS_APP_ID)) { _cosOptions.Appid = options[COS_APP_ID]; } if (options.ContainsKey(COS_REGION)) { _cosOptions.Region = options[COS_REGION]; } if (options.ContainsKey(COS_SECRET_ID)) { _cosOptions.SecretId = options[COS_SECRET_ID]; } if (options.ContainsKey(COS_SECRET_KEY)) { _cosOptions.SecretKey = options[COS_SECRET_KEY]; } if (options.ContainsKey(COS_BUCKET)) { _cosOptions.Bucket = options[COS_BUCKET]; } if (options.ContainsKey(COS_STORAGE_CLASS)) { _cosOptions.StorageClass = options[COS_STORAGE_CLASS]; } } CosXml GetCosXml() { string appid = _cosOptions.Appid; string region = _cosOptions.Region; string secretId = _cosOptions.SecretId; string secretKey = _cosOptions.SecretKey; CosXmlConfig config = new CosXmlConfig.Builder() .SetConnectionTimeoutMs(CONNECTION_TIMEOUT_MS) .SetReadWriteTimeoutMs(READ_WRITE_TIMEOUT_MS) .IsHttps(COS_USE_SSL) .SetAppid(appid) .SetRegion(region) .Build(); // Initialization QCloudCredentialProvider, COS SDK provides three ways: the permanent temporary keys custom QCloudCredentialProvider cosCredentialProvider = new DefaultQCloudCredentialProvider(secretId, secretKey, KEY_DURATION_SECOND); return new CosXmlServer(config, cosCredentialProvider); } public IEnumerable List() { bool isTruncated = true; string filename = null; while (isTruncated) { cosXml = GetCosXml(); string bucket = _cosOptions.Bucket; string prefix = _cosOptions.Path; GetBucketRequest request = new GetBucketRequest(bucket); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); if (!string.IsNullOrEmpty(filename)) { request.SetMarker(filename); } if (!string.IsNullOrEmpty(prefix)) { request.SetPrefix(prefix); } GetBucketResult result = cosXml.GetBucket(request); ListBucket info = result.listBucket; isTruncated = result.listBucket.isTruncated; filename = result.listBucket.nextMarker; foreach (var item in info.contentsList) { var last = DateTime.Parse(item.lastModified); var fileName = item.key; if (!string.IsNullOrWhiteSpace(prefix)) { fileName = fileName.Substring(prefix.Length); if (fileName.StartsWith("/", StringComparison.Ordinal)) { fileName = fileName.Trim('/'); } } yield return new FileEntry(fileName, item.size, last, last); } } } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) await PutAsync(remotename, fs, cancelToken); } public void Get(string remotename, string filename) { using (var fs = File.Open(filename, FileMode.Create, FileAccess.Write, FileShare.None)) Get(remotename, fs); } public void Delete(string remotename) { try { cosXml = GetCosXml(); string bucket = _cosOptions.Bucket; string key = GetFullKey(remotename); DeleteObjectRequest request = new DeleteObjectRequest(bucket, key); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); DeleteObjectResult result = cosXml.DeleteObject(request); } catch (COSXML.CosException.CosClientException clientEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Delete", clientEx, "Delete failed: {0}", remotename); throw; } catch (COSXML.CosException.CosServerException serverEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Delete", serverEx, "Delete failed: {0}, {1}", remotename, serverEx.GetInfo()); throw; } } public void Test() { var json = JsonConvert.SerializeObject(_cosOptions); try { cosXml = GetCosXml(); string bucket = _cosOptions.Bucket; HeadBucketRequest request = new HeadBucketRequest(bucket); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); HeadBucketResult result = cosXml.HeadBucket(request); Logging.Log.WriteInformationMessage(LOGTAG, "Test", "Request complete {0}: {1}, {2}", result.httpCode, json, result.GetResultInfo()); } catch (COSXML.CosException.CosClientException clientEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Test", clientEx, "Request failed: {0}", json); throw; } catch (COSXML.CosException.CosServerException serverEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Test", serverEx, "Request failed: {0}, {1}", json, serverEx.GetInfo()); throw; } } public void CreateFolder() { // No need to create folders } public void Dispose() { if (cosXml != null) { cosXml = null; } } public Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { cosXml = GetCosXml(); string bucket = _cosOptions.Bucket; string key = GetFullKey(remotename); byte[] buffer = new byte[stream.Length]; if (Utility.Utility.ForceStreamRead(stream, buffer, buffer.Length) != stream.Length) { throw new Exception("Bad file read"); } PutObjectRequest request = new PutObjectRequest(bucket, key, buffer); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), KEY_DURATION_SECOND); request.SetRequestHeader("Content-Type", "application/octet-stream"); if (!string.IsNullOrEmpty(_cosOptions.StorageClass)) { request.SetRequestHeader("x-" + COS_STORAGE_CLASS, _cosOptions.StorageClass); } PutObjectResult result = cosXml.PutObject(request); } catch (COSXML.CosException.CosClientException clientEx) { Logging.Log.WriteErrorMessage(LOGTAG, "PutAsync", clientEx, "Put failed: {0}", remotename); throw; } catch (COSXML.CosException.CosServerException serverEx) { Logging.Log.WriteErrorMessage(LOGTAG, "PutAsync", serverEx, "Put failed: {0}, {1}", remotename, serverEx.GetInfo()); throw; } return Task.CompletedTask; } public void Get(string remotename, Stream stream) { try { cosXml = GetCosXml(); string bucket = _cosOptions.Bucket; string key = GetFullKey(remotename); GetObjectBytesRequest request = new GetObjectBytesRequest(bucket, key); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), KEY_DURATION_SECOND); GetObjectBytesResult result = cosXml.GetObject(request); byte[] bytes = result.content; Stream ms = new MemoryStream(bytes); Utility.Utility.CopyStream(ms, stream); } catch (COSXML.CosException.CosClientException clientEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Get", clientEx, "Get failed: {0}", remotename); throw; } catch (COSXML.CosException.CosServerException serverEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Get", serverEx, "Get failed: {0}, {1}", remotename, serverEx.GetInfo()); throw; } } public void Rename(string oldname, string newname) { try { cosXml = GetCosXml(); string sourceAppid = _cosOptions.Appid; string sourceBucket = _cosOptions.Bucket; string sourceRegion = _cosOptions.Region; string sourceKey = GetFullKey(oldname); CopySourceStruct copySource = new CopySourceStruct(sourceAppid, sourceBucket, sourceRegion, sourceKey); string bucket = _cosOptions.Bucket; string key = GetFullKey(newname); CopyObjectRequest request = new CopyObjectRequest(bucket, key); request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), KEY_DURATION_SECOND); request.SetCopySource(copySource); request.SetCopyMetaDataDirective(COSXML.Common.CosMetaDataDirective.COPY); CopyObjectResult result = cosXml.CopyObject(request); //Console.WriteLine(result.GetResultInfo()); Delete(oldname); } catch (COSXML.CosException.CosClientException clientEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Rename", clientEx, "Rename failed: {0} to {1}", oldname, newname); throw; } catch (COSXML.CosException.CosServerException serverEx) { Logging.Log.WriteErrorMessage(LOGTAG, "Rename", serverEx, "Rename failed: {0} to {1}, {2}", oldname, newname, serverEx.GetInfo()); throw; } } public string DisplayName => Strings.COSBackend.DisplayName; public string Description => Strings.COSBackend.Description; public string ProtocolKey => "cos"; public IList SupportedCommands { get { return new List(new ICommandLineArgument[] { new CommandLineArgument(COS_APP_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAccountDescriptionShort, Strings.COSBackend.COSAccountDescriptionLong), new CommandLineArgument(COS_REGION, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSLocationDescriptionShort, Strings.COSBackend.COSLocationDescriptionLong), new CommandLineArgument(COS_SECRET_ID, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSAPISecretIdDescriptionShort, Strings.COSBackend.COSAPISecretIdDescriptionLong), new CommandLineArgument(COS_SECRET_KEY, CommandLineArgument.ArgumentType.Password, Strings.COSBackend.COSAPISecretKeyDescriptionShort, Strings.COSBackend.COSAPISecretKeyDescriptionLong), new CommandLineArgument(COS_BUCKET, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSBucketDescriptionShort, Strings.COSBackend.COSBucketDescriptionLong), new CommandLineArgument(COS_STORAGE_CLASS, CommandLineArgument.ArgumentType.String, Strings.COSBackend.COSStorageClassDescriptionShort, Strings.COSBackend.COSStorageClassDescriptionLong) }); } } public string[] DNSName => null; private string GetFullKey(string name) { if (string.IsNullOrWhiteSpace(_cosOptions?.Path)) { return name; } return _cosOptions.Path + name; } } }