302 lines
12 KiB
C#
302 lines
12 KiB
C#
// Copyright (C) 2024, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using Duplicati.Library.Backend.GoogleServices;
|
|
using Duplicati.Library.Common.IO;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Utility;
|
|
using Newtonsoft.Json;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace Duplicati.Library.Backend.GoogleCloudStorage
|
|
{
|
|
// ReSharper disable once UnusedMember.Global
|
|
// This class is instantiated dynamically in the BackendLoader.
|
|
public class GoogleCloudStorage : IBackend, IStreamingBackend, IRenameEnabledBackend
|
|
{
|
|
private const string AUTHID_OPTION = "authid";
|
|
private const string PROJECT_OPTION = "gcs-project";
|
|
|
|
private const string LOCATION_OPTION = "gcs-location";
|
|
private const string STORAGECLASS_OPTION = "gcs-storage-class";
|
|
|
|
private readonly string m_bucket;
|
|
private readonly string m_prefix;
|
|
private readonly string m_project;
|
|
private readonly OAuthHelper m_oauth;
|
|
|
|
private readonly string m_location;
|
|
private readonly string m_storage_class;
|
|
|
|
public GoogleCloudStorage()
|
|
{
|
|
}
|
|
|
|
public GoogleCloudStorage(string url, Dictionary<string, string> options)
|
|
{
|
|
var uri = new Utility.Uri(url);
|
|
|
|
m_bucket = uri.Host;
|
|
m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/");
|
|
|
|
// For GCS we do not use a leading slash
|
|
if (m_prefix.StartsWith("/", StringComparison.Ordinal))
|
|
m_prefix = m_prefix.Substring(1);
|
|
|
|
string authid;
|
|
options.TryGetValue(AUTHID_OPTION, out authid);
|
|
options.TryGetValue(PROJECT_OPTION, out m_project);
|
|
options.TryGetValue(LOCATION_OPTION, out m_location);
|
|
options.TryGetValue(STORAGECLASS_OPTION, out m_storage_class);
|
|
|
|
if (string.IsNullOrEmpty(authid))
|
|
throw new UserInformationException(Strings.GoogleCloudStorage.MissingAuthID(AUTHID_OPTION), "GoogleCloudStorageMissingAuthID");
|
|
|
|
m_oauth = new OAuthHelper(authid, this.ProtocolKey);
|
|
m_oauth.AutoAuthHeader = true;
|
|
}
|
|
|
|
|
|
private class ListBucketResponse
|
|
{
|
|
public string nextPageToken { get; set; }
|
|
public BucketResourceItem[] items { get; set; }
|
|
}
|
|
|
|
private class BucketResourceItem
|
|
{
|
|
public string name { get; set; }
|
|
public DateTime? updated { get; set; }
|
|
public long? size { get; set; }
|
|
}
|
|
|
|
private class CreateBucketRequest
|
|
{
|
|
public string name { get; set; }
|
|
public string location { get; set; }
|
|
public string storageClass { get; set; }
|
|
}
|
|
|
|
private T HandleListExceptions<T>(Func<T> func)
|
|
{
|
|
try
|
|
{
|
|
return func();
|
|
}
|
|
catch (WebException wex)
|
|
{
|
|
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
|
|
throw new FolderMissingException();
|
|
else
|
|
throw;
|
|
}
|
|
}
|
|
|
|
#region IBackend implementation
|
|
public IEnumerable<IFileEntry> List()
|
|
{
|
|
var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix));
|
|
while (true)
|
|
{
|
|
var resp = HandleListExceptions(() => m_oauth.ReadJSONResponse<ListBucketResponse>(url));
|
|
|
|
if (resp.items != null)
|
|
foreach (var f in resp.items)
|
|
{
|
|
var name = f.name;
|
|
if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
|
|
name = name.Substring(m_prefix.Length);
|
|
if (f.size == null)
|
|
yield return new FileEntry(name);
|
|
else if (f.updated == null)
|
|
yield return new FileEntry(name, f.size.Value);
|
|
else
|
|
yield return new FileEntry(name, f.size.Value, f.updated.Value, f.updated.Value);
|
|
}
|
|
|
|
var token = resp.nextPageToken;
|
|
if (string.IsNullOrWhiteSpace(token))
|
|
break;
|
|
url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix), token);
|
|
}
|
|
}
|
|
|
|
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
|
|
await PutAsync(remotename, fs, cancelToken);
|
|
}
|
|
|
|
public void Get(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.Create(filename))
|
|
Get(remotename, fs);
|
|
}
|
|
public void Delete(string remotename)
|
|
{
|
|
|
|
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)));
|
|
req.Method = "DELETE";
|
|
|
|
m_oauth.ReadJSONResponse<object>(req);
|
|
}
|
|
|
|
public void Test()
|
|
{
|
|
this.TestList();
|
|
}
|
|
|
|
public void CreateFolder()
|
|
{
|
|
if (string.IsNullOrEmpty(m_project))
|
|
throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID");
|
|
|
|
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
|
|
{
|
|
name = m_bucket,
|
|
location = m_location,
|
|
storageClass = m_storage_class
|
|
}));
|
|
|
|
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.CreateFolderUrl(m_project));
|
|
req.Method = "POST";
|
|
req.ContentLength = data.Length;
|
|
req.ContentType = "application/json; charset=UTF-8";
|
|
|
|
var areq = new AsyncHttpRequest(req);
|
|
|
|
using (var rs = areq.GetRequestStream())
|
|
rs.Write(data, 0, data.Length);
|
|
|
|
m_oauth.ReadJSONResponse<BucketResourceItem>(areq);
|
|
}
|
|
|
|
public string DisplayName
|
|
{
|
|
get { return Strings.GoogleCloudStorage.DisplayName; }
|
|
}
|
|
|
|
public string ProtocolKey
|
|
{
|
|
get { return "gcs"; }
|
|
}
|
|
|
|
public IList<ICommandLineArgument> SupportedCommands
|
|
{
|
|
get
|
|
{
|
|
StringBuilder locations = new StringBuilder();
|
|
StringBuilder storageClasses = new StringBuilder();
|
|
|
|
foreach (KeyValuePair<string, string> s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS)
|
|
locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
|
|
foreach (KeyValuePair<string, string> s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES)
|
|
storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
|
|
|
|
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
|
|
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())),
|
|
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(storageClasses.ToString())),
|
|
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleCloudStorage.AuthidShort, Strings.GoogleCloudStorage.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("gcs"))),
|
|
new CommandLineArgument(PROJECT_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.ProjectDescriptionShort, Strings.GoogleCloudStorage.ProjectDescriptionLong),
|
|
});
|
|
}
|
|
}
|
|
public string Description
|
|
{
|
|
get { return Strings.GoogleCloudStorage.Description; }
|
|
}
|
|
|
|
public string[] DNSName
|
|
{
|
|
get
|
|
{
|
|
return WebApi.GoogleCloudStorage.Hosts();
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
|
|
{
|
|
var item = new BucketResourceItem { name = m_prefix + remotename };
|
|
|
|
var url = WebApi.GoogleCloudStorage.PutUrl(m_bucket);
|
|
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, cancelToken);
|
|
|
|
if (res == null)
|
|
throw new Exception("Upload succeeded, but no data was returned");
|
|
}
|
|
|
|
public void Get(string remotename, System.IO.Stream stream)
|
|
{
|
|
try
|
|
{
|
|
var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
var req = m_oauth.CreateRequest(url);
|
|
var areq = new AsyncHttpRequest(req);
|
|
|
|
using (var resp = areq.GetResponse())
|
|
using (var rs = areq.GetResponseStream())
|
|
Library.Utility.Utility.CopyStream(rs, stream);
|
|
}
|
|
catch (WebException wex)
|
|
{
|
|
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
|
|
throw new FileMissingException();
|
|
else
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public void Rename(string oldname, string newname)
|
|
{
|
|
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
|
|
{
|
|
name = m_prefix + newname,
|
|
}));
|
|
|
|
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.RenameUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + oldname)));
|
|
req.Method = "PATCH";
|
|
req.ContentLength = data.Length;
|
|
req.ContentType = "application/json; charset=UTF-8";
|
|
|
|
var areq = new AsyncHttpRequest(req);
|
|
using (var rs = areq.GetRequestStream())
|
|
rs.Write(data, 0, data.Length);
|
|
|
|
m_oauth.ReadJSONResponse<BucketResourceItem>(req);
|
|
}
|
|
|
|
#region IDisposable implementation
|
|
public void Dispose()
|
|
{
|
|
|
|
}
|
|
#endregion
|
|
}
|
|
}
|
|
|