irc-netcore-api/IRC.Core.SCP/Service/OSSService.cs

771 lines
26 KiB
C#
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

using AlibabaCloud.SDK.Sts20150401;
using Aliyun.OSS;
using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using Amazon.SecurityToken;
using Amazon.SecurityToken.Model;
using IRaCIS.Core.Infrastructure;
using IRaCIS.Core.Infrastructure.NewtonsoftJson;
using MassTransit;
using Microsoft.Extensions.Options;
using Minio;
using Minio.DataModel.Args;
using System.Reactive.Linq;
using System.Runtime.InteropServices;
namespace IRaCIS.Core.SCP;
#region 绑定和返回模型
[LowerCamelCaseJson]
public class MinIOOptions : AWSOptions
{
public int Port { get; set; }
}
public class AWSOptions
{
public string EndPoint { get; set; }
public bool UseSSL { get; set; }
public string AccessKeyId { get; set; }
public string RoleArn { get; set; }
public string SecretAccessKey { get; set; }
public string BucketName { get; set; }
public string ViewEndpoint { get; set; }
public int DurationSeconds { get; set; }
public string Region { get; set; }
}
public class AliyunOSSOptions
{
public string RegionId { get; set; }
public string AccessKeyId { get; set; }
public string AccessKeySecret { get; set; }
public string InternalEndpoint { get; set; }
public string EndPoint { get; set; }
public string BucketName { get; set; }
public string RoleArn { get; set; }
public string Region { get; set; }
public string ViewEndpoint { get; set; }
public int DurationSeconds { get; set; }
}
public class ObjectStoreServiceOptions
{
public string ObjectStoreUse { get; set; }
public AliyunOSSOptions AliyunOSS { get; set; }
public MinIOOptions MinIO { get; set; }
public AWSOptions AWS { get; set; }
}
public class ObjectStoreDTO
{
public string ObjectStoreUse { get; set; }
public AliyunOSSTempToken AliyunOSS { get; set; }
public MinIOOptions MinIO { get; set; }
public AWSTempToken AWS { get; set; }
}
[LowerCamelCaseJson]
public class AliyunOSSTempToken
{
public string AccessKeyId { get; set; }
public string AccessKeySecret { get; set; }
public string EndPoint { get; set; }
public string BucketName { get; set; }
public string Region { get; set; }
public string ViewEndpoint { get; set; }
public string SecurityToken { get; set; }
public DateTime Expiration { get; set; }
}
[LowerCamelCaseJson]
public class AWSTempToken
{
public string Region { get; set; }
public string SessionToken { get; set; }
public string EndPoint { get; set; }
public string AccessKeyId { get; set; }
public string SecretAccessKey { get; set; }
public string BucketName { get; set; }
public string ViewEndpoint { get; set; }
public DateTime Expiration { get; set; }
}
public enum ObjectStoreUse
{
AliyunOSS = 0,
MinIO = 1,
AWS = 2,
}
#endregion
// aws 参考链接 https://github.com/awsdocs/aws-doc-sdk-examples/tree/main/dotnetv3/S3/S3_Basics
public interface IOSSService
{
public Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true);
public Task<string> UploadToOSSAsync(string localFilePath, string oosFolderPath, bool isFileNameAddGuid = true);
public Task DownLoadFromOSSAsync(string ossRelativePath, string localFilePath);
public ObjectStoreServiceOptions ObjectStoreServiceOptions { get; set; }
public Task<string> GetSignedUrl(string ossRelativePath);
public Task DeleteFromPrefix(string prefix);
public ObjectStoreDTO GetObjectStoreTempToken();
}
public class OSSService : IOSSService
{
public ObjectStoreServiceOptions ObjectStoreServiceOptions { get; set; }
private AliyunOSSTempToken AliyunOSSTempToken { get; set; }
private AWSTempToken AWSTempToken { get; set; }
public OSSService(IOptionsMonitor<ObjectStoreServiceOptions> options)
{
ObjectStoreServiceOptions = options.CurrentValue;
}
/// <summary>
/// oosFolderPath 不要 "/ "开头 应该: TempFolder/ChildFolder
/// </summary>
/// <param name="fileStream"></param>
/// <param name="oosFolderPath"></param>
/// <param name="fileRealName"></param>
/// <param name="isFileNameAddGuid"></param>
/// <returns></returns>
public async Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true)
{
GetObjectStoreTempToken();
var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}" : $"{oosFolderPath}/{fileRealName}";
try
{
using (var memoryStream = new MemoryStream())
{
fileStream.Seek(0, SeekOrigin.Begin);
fileStream.CopyTo(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin);
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
var _ossClient = new OssClient(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? aliConfig.EndPoint : aliConfig.InternalEndpoint, AliyunOSSTempToken.AccessKeyId, AliyunOSSTempToken.AccessKeySecret, AliyunOSSTempToken.SecurityToken);
// 上传文件
var result = _ossClient.PutObject(aliConfig.BucketName, ossRelativePath, memoryStream);
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "MinIO")
{
var minIOConfig = ObjectStoreServiceOptions.MinIO;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}:{minIOConfig.Port}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
var putObjectArgs = new PutObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithStreamData(memoryStream)
.WithObjectSize(memoryStream.Length);
await minioClient.PutObjectAsync(putObjectArgs);
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var awsConfig = ObjectStoreServiceOptions.AWS;
var credentials = new SessionAWSCredentials(AWSTempToken.AccessKeyId, AWSTempToken.SecretAccessKey, AWSTempToken.SessionToken);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1,
UseHttp = true,
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var putObjectRequest = new Amazon.S3.Model.PutObjectRequest()
{
BucketName = awsConfig.BucketName,
InputStream = memoryStream,
Key = ossRelativePath,
};
await amazonS3Client.PutObjectAsync(putObjectRequest);
}
else
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
}
}
}
catch (Exception ex)
{
throw new BusinessValidationFailedException($"上传发生异常:{ex.Message}");
}
return "/" + ossRelativePath;
}
/// <summary>
/// oosFolderPath 不要 "/ "开头 应该: TempFolder/ChildFolder
/// </summary>
/// <param name="localFilePath"></param>
/// <param name="oosFolderPath"></param>
/// <param name="isFileNameAddGuid"></param>
/// <returns></returns>
/// <exception cref="BusinessValidationFailedException"></exception>
public async Task<string> UploadToOSSAsync(string localFilePath, string oosFolderPath, bool isFileNameAddGuid = true)
{
GetObjectStoreTempToken();
var localFileName = Path.GetFileName(localFilePath);
var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{localFileName}" : $"{oosFolderPath}/{localFileName}";
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
var _ossClient = new OssClient(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? aliConfig.EndPoint : aliConfig.InternalEndpoint, AliyunOSSTempToken.AccessKeyId, AliyunOSSTempToken.AccessKeySecret, AliyunOSSTempToken.SecurityToken);
// 上传文件
var result = _ossClient.PutObject(aliConfig.BucketName, ossRelativePath, localFilePath);
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "MinIO")
{
var minIOConfig = ObjectStoreServiceOptions.MinIO;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}:{minIOConfig.Port}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
var putObjectArgs = new PutObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithFileName(localFilePath);
await minioClient.PutObjectAsync(putObjectArgs);
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new SessionAWSCredentials(AWSTempToken.AccessKeyId, AWSTempToken.SecretAccessKey, AWSTempToken.SessionToken);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1,
UseHttp = true,
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var putObjectRequest = new Amazon.S3.Model.PutObjectRequest()
{
BucketName = awsConfig.BucketName,
FilePath = localFilePath,
Key = ossRelativePath,
};
await amazonS3Client.PutObjectAsync(putObjectRequest);
}
else
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
}
return "/" + ossRelativePath;
}
public async Task DownLoadFromOSSAsync(string ossRelativePath, string localFilePath)
{
GetObjectStoreTempToken();
ossRelativePath = ossRelativePath.TrimStart('/');
try
{
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
var _ossClient = new OssClient(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? aliConfig.EndPoint : aliConfig.InternalEndpoint, AliyunOSSTempToken.AccessKeyId, AliyunOSSTempToken.AccessKeySecret, AliyunOSSTempToken.SecurityToken);
// 上传文件
var result = _ossClient.GetObject(aliConfig.BucketName, ossRelativePath);
// 将下载的文件流保存到本地文件
using (var fs = File.OpenWrite(localFilePath))
{
result.Content.CopyTo(fs);
fs.Close();
}
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "MinIO")
{
var minIOConfig = ObjectStoreServiceOptions.MinIO;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}:{minIOConfig.Port}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
var getObjectArgs = new GetObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithFile(localFilePath);
await minioClient.GetObjectAsync(getObjectArgs);
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new SessionAWSCredentials(AWSTempToken.AccessKeyId, AWSTempToken.SecretAccessKey, AWSTempToken.SessionToken);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1,
UseHttp = true,
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var getObjectArgs = new Amazon.S3.Model.GetObjectRequest()
{
BucketName = awsConfig.BucketName,
Key = ossRelativePath,
};
await (await amazonS3Client.GetObjectAsync(getObjectArgs)).WriteResponseStreamToFileAsync(localFilePath, true, CancellationToken.None);
}
else
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
}
}
catch (Exception ex)
{
throw new BusinessValidationFailedException("oss下载失败!" + ex.Message);
}
}
public async Task<string> GetSignedUrl(string ossRelativePath)
{
GetObjectStoreTempToken();
ossRelativePath = ossRelativePath.TrimStart('/');
try
{
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
var _ossClient = new OssClient(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? aliConfig.EndPoint : aliConfig.InternalEndpoint, AliyunOSSTempToken.AccessKeyId, AliyunOSSTempToken.AccessKeySecret, AliyunOSSTempToken.SecurityToken);
// 生成签名URL。
var req = new GeneratePresignedUriRequest(aliConfig.BucketName, ossRelativePath, SignHttpMethod.Get)
{
// 设置签名URL过期时间默认值为3600秒。
Expiration = DateTime.Now.AddHours(1),
};
var uri = _ossClient.GeneratePresignedUri(req);
return uri.PathAndQuery;
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "MinIO")
{
var minIOConfig = ObjectStoreServiceOptions.MinIO;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}:{minIOConfig.Port}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
var args = new PresignedGetObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithExpiry(3600)
/*.WithHeaders(reqParams)*/;
var presignedUrl = await minioClient.PresignedGetObjectAsync(args);
Uri uri = new Uri(presignedUrl);
string relativePath = uri.PathAndQuery;
return relativePath;
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new SessionAWSCredentials(AWSTempToken.AccessKeyId, AWSTempToken.SecretAccessKey, AWSTempToken.SessionToken);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1,
UseHttp = true,
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var presignedUrl = await amazonS3Client.GetPreSignedURLAsync(new GetPreSignedUrlRequest()
{
BucketName = awsConfig.BucketName,
Key = ossRelativePath,
Expires = DateTime.UtcNow.AddMinutes(120)
});
Uri uri = new Uri(presignedUrl);
string relativePath = uri.PathAndQuery;
return relativePath;
}
else
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
}
}
catch (Exception ex)
{
throw new BusinessValidationFailedException("oss授权url失败!" + ex.Message);
}
}
/// <summary>
/// 删除某个目录的文件
/// </summary>
/// <param name="prefix"></param>
/// <returns></returns>
public async Task DeleteFromPrefix(string prefix)
{
GetObjectStoreTempToken();
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
var _ossClient = new OssClient(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? aliConfig.EndPoint : aliConfig.InternalEndpoint, AliyunOSSTempToken.AccessKeyId, AliyunOSSTempToken.AccessKeySecret, AliyunOSSTempToken.SecurityToken);
try
{
ObjectListing objectListing = null;
string nextMarker = null;
do
{
// 使用 prefix 模拟目录结构,设置 MaxKeys 和 NextMarker
objectListing = _ossClient.ListObjects(new Aliyun.OSS.ListObjectsRequest(aliConfig.BucketName)
{
Prefix = prefix,
MaxKeys = 1000,
Marker = nextMarker
});
List<string> keys = objectListing.ObjectSummaries.Select(t => t.Key).ToList();
// 删除获取到的文件
if (keys.Count > 0)
{
_ossClient.DeleteObjects(new Aliyun.OSS.DeleteObjectsRequest(aliConfig.BucketName, keys, false));
}
// 设置 NextMarker 以获取下一页的数据
nextMarker = objectListing.NextMarker;
} while (objectListing.IsTruncated);
}
catch (Exception ex)
{
Console.WriteLine($"Error: {ex.Message}");
}
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "MinIO")
{
var minIOConfig = ObjectStoreServiceOptions.MinIO;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}:{minIOConfig.Port}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
var listArgs = new ListObjectsArgs().WithBucket(minIOConfig.BucketName).WithPrefix(prefix).WithRecursive(true);
// 创建一个空列表用于存储对象键
var objects = new List<string>();
// 使用 await foreach 来异步迭代对象列表
await foreach (var item in minioClient.ListObjectsEnumAsync(listArgs))
{
objects.Add(item.Key);
}
if (objects.Count > 0)
{
var objArgs = new RemoveObjectsArgs()
.WithBucket(minIOConfig.BucketName)
.WithObjects(objects);
// 删除对象
await minioClient.RemoveObjectsAsync(objArgs);
}
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new SessionAWSCredentials(AWSTempToken.AccessKeyId, AWSTempToken.SecretAccessKey, AWSTempToken.SessionToken);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1,
UseHttp = true,
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
// 列出指定前缀下的所有对象
var listObjectsRequest = new ListObjectsV2Request
{
BucketName = awsConfig.BucketName,
Prefix = prefix
};
var listObjectsResponse = await amazonS3Client.ListObjectsV2Async(listObjectsRequest);
if (listObjectsResponse.S3Objects.Count > 0)
{
// 准备删除请求
var deleteObjectsRequest = new Amazon.S3.Model.DeleteObjectsRequest
{
BucketName = awsConfig.BucketName,
Objects = new List<KeyVersion>()
};
foreach (var s3Object in listObjectsResponse.S3Objects)
{
deleteObjectsRequest.Objects.Add(new KeyVersion
{
Key = s3Object.Key
});
}
// 批量删除对象
var deleteObjectsResponse = await amazonS3Client.DeleteObjectsAsync(deleteObjectsRequest);
}
}
else
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
}
}
public ObjectStoreDTO GetObjectStoreTempToken()
{
var ossOptions = ObjectStoreServiceOptions.AliyunOSS;
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var client = new Client(new AlibabaCloud.OpenApiClient.Models.Config()
{
AccessKeyId = ossOptions.AccessKeyId,
AccessKeySecret = ossOptions.AccessKeySecret,
//AccessKeyId = "LTAI5tJV76pYX5yPg1N9QVE8",
//AccessKeySecret = "roRNLa9YG1of4pYruJGCNKBXEWTAWa",
Endpoint = "sts.cn-hangzhou.aliyuncs.com"
});
var assumeRoleRequest = new AlibabaCloud.SDK.Sts20150401.Models.AssumeRoleRequest();
// 将<YOUR_ROLE_SESSION_NAME>设置为自定义的会话名称例如oss-role-session。
assumeRoleRequest.RoleSessionName = $"session-name-{NewId.NextGuid()}";
// 将<YOUR_ROLE_ARN>替换为拥有上传文件到指定OSS Bucket权限的RAM角色的ARN。
assumeRoleRequest.RoleArn = ossOptions.RoleArn;
//assumeRoleRequest.RoleArn = "acs:ram::1899121822495495:role/webdirect";
assumeRoleRequest.DurationSeconds = ossOptions.DurationSeconds;
var runtime = new AlibabaCloud.TeaUtil.Models.RuntimeOptions();
var response = client.AssumeRoleWithOptions(assumeRoleRequest, runtime);
var credentials = response.Body.Credentials;
var tempToken = new AliyunOSSTempToken()
{
AccessKeyId = credentials.AccessKeyId,
AccessKeySecret = credentials.AccessKeySecret,
//转为服务器时区,最后统一转为客户端时区
Expiration = TimeZoneInfo.ConvertTimeFromUtc(DateTime.Parse(credentials.Expiration), TimeZoneInfo.Local),
SecurityToken = credentials.SecurityToken,
Region = ossOptions.Region,
BucketName = ossOptions.BucketName,
EndPoint = ossOptions.EndPoint,
ViewEndpoint = ossOptions.ViewEndpoint,
};
AliyunOSSTempToken = tempToken;
return new ObjectStoreDTO() { ObjectStoreUse = ObjectStoreServiceOptions.ObjectStoreUse, AliyunOSS = tempToken };
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "MinIO")
{
return new ObjectStoreDTO() { ObjectStoreUse = ObjectStoreServiceOptions.ObjectStoreUse, MinIO = ObjectStoreServiceOptions.MinIO };
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var awsOptions = ObjectStoreServiceOptions.AWS;
//aws 临时凭证
// 创建 STS 客户端
var stsClient = new AmazonSecurityTokenServiceClient(awsOptions.AccessKeyId, awsOptions.SecretAccessKey);
// 使用 AssumeRole 请求临时凭证
var assumeRoleRequest = new AssumeRoleRequest
{
RoleArn = awsOptions.RoleArn, // 角色 ARN
RoleSessionName = $"session-name-{NewId.NextGuid()}",
DurationSeconds = awsOptions.DurationSeconds // 临时凭证有效期
};
var assumeRoleResponse = stsClient.AssumeRoleAsync(assumeRoleRequest).Result;
var credentials = assumeRoleResponse.Credentials;
var tempToken = new AWSTempToken()
{
AccessKeyId = credentials.AccessKeyId,
SecretAccessKey = credentials.SecretAccessKey,
SessionToken = credentials.SessionToken,
Expiration = credentials.Expiration,
Region = awsOptions.Region,
BucketName = awsOptions.BucketName,
EndPoint = awsOptions.EndPoint,
ViewEndpoint = awsOptions.ViewEndpoint,
};
AWSTempToken = tempToken;
return new ObjectStoreDTO() { ObjectStoreUse = ObjectStoreServiceOptions.ObjectStoreUse, AWS = tempToken };
}
else
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
}
}
}