s3 增加region

IRC_NewDev
hang 2024-09-04 11:04:39 +08:00
parent cec6174bb6
commit f4426dd280
4 changed files with 122 additions and 39 deletions

View File

@ -380,6 +380,7 @@ namespace IRaCIS.Api.Controllers
SessionToken= credentials.SessionToken,
Expiration=credentials.Expiration,
BucketName = awsOptions.BucketName,
EndPoint = awsOptions.EndPoint,
ViewEndpoint = awsOptions.ViewEndpoint,

View File

@ -37,6 +37,7 @@
"ViewEndpoint": "https://hir-oss.test.extimaging.com/irc-test"
},
"AWS": {
"Region": "us-east-1",
"EndPoint": "s3.us-east-1.amazonaws.com",
"UseSSL": true,
"RoleArn": "arn:aws:iam::471112624751:role/sts_s3_upload",

View File

@ -20,9 +20,15 @@ using System.Reactive.Linq;
using IRaCIS.Core.Domain.Share;
using static MassTransit.ValidationResultExtensions;
using IRaCIS.Core.Infrastructure.NewtonsoftJson;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using MassTransit.Caching.Internals;
namespace IRaCIS.Core.Application.Helper
{
#region 绑定和返回模型
[LowerCamelCaseJson]
public class MinIOOptions : AWSOptions
{
@ -30,7 +36,7 @@ namespace IRaCIS.Core.Application.Helper
}
public class AWSOptions
{
public string EndPoint { get; set; }
@ -41,6 +47,7 @@ namespace IRaCIS.Core.Application.Helper
public string BucketName { get; set; }
public string ViewEndpoint { get; set; }
public int DurationSeconds { get; set; }
public string Region { get; set; }
}
public class AliyunOSSOptions
@ -118,6 +125,7 @@ namespace IRaCIS.Core.Application.Helper
[LowerCamelCaseJson]
public class AWSTempToken
{
public string Region { get; set; }
public string SessionToken { get; set; }
public string EndPoint { get; set; }
public string AccessKeyId { get; set; }
@ -134,6 +142,10 @@ namespace IRaCIS.Core.Application.Helper
AWS = 2,
}
#endregion
// aws 参考链接 https://github.com/awsdocs/aws-doc-sdk-examples/tree/main/dotnetv3/S3/S3_Basics
public interface IOSSService
{
public Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true);
@ -170,8 +182,6 @@ namespace IRaCIS.Core.Application.Helper
public async Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true)
{
var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}" : $"{oosFolderPath}/{fileRealName}";
//var ossRelativePath = $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}";
//var ossRelativePath = oosFolderPath + "/" + fileRealName;
try
{
@ -215,20 +225,27 @@ namespace IRaCIS.Core.Application.Helper
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var minIOConfig = ObjectStoreServiceOptions.AWS;
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
ServiceURL = awsConfig.EndPoint
};
var putObjectArgs = new PutObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithStreamData(memoryStream)
.WithObjectSize(memoryStream.Length);
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
await minioClient.PutObjectAsync(putObjectArgs);
var putObjectRequest = new Amazon.S3.Model.PutObjectRequest()
{
BucketName = awsConfig.BucketName,
InputStream = memoryStream,
Key = ossRelativePath,
};
await amazonS3Client.PutObjectAsync(putObjectRequest);
}
else
{
@ -256,6 +273,7 @@ namespace IRaCIS.Core.Application.Helper
/// </summary>
/// <param name="localFilePath"></param>
/// <param name="oosFolderPath"></param>
/// <param name="isFileNameAddGuid"></param>
/// <returns></returns>
/// <exception cref="BusinessValidationFailedException"></exception>
public async Task<string> UploadToOSSAsync(string localFilePath, string oosFolderPath, bool isFileNameAddGuid = true)
@ -265,9 +283,6 @@ namespace IRaCIS.Core.Application.Helper
var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{localFileName}" : $"{oosFolderPath}/{localFileName}";
//var ossRelativePath = oosFolderPath + "/" + localFileName;
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{
var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
@ -296,19 +311,28 @@ namespace IRaCIS.Core.Application.Helper
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var minIOConfig = ObjectStoreServiceOptions.AWS;
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
ServiceURL = awsConfig.EndPoint
};
var putObjectArgs = new PutObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithFileName(localFilePath);
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var putObjectRequest = new Amazon.S3.Model.PutObjectRequest()
{
BucketName = awsConfig.BucketName,
FilePath = localFilePath,
Key = ossRelativePath,
};
await amazonS3Client.PutObjectAsync(putObjectRequest);
await minioClient.PutObjectAsync(putObjectArgs);
}
else
{
@ -321,7 +345,7 @@ namespace IRaCIS.Core.Application.Helper
public async Task DownLoadFromOSSAsync(string ossRelativePath, string localFilePath)
{
ossRelativePath = ossRelativePath.TrimStart('/');
try
{
@ -362,18 +386,29 @@ namespace IRaCIS.Core.Application.Helper
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
var minIOConfig = ObjectStoreServiceOptions.AWS;
var awsConfig = ObjectStoreServiceOptions.AWS;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}")
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL)
.Build();
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
ServiceURL = awsConfig.EndPoint
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var getObjectArgs = new Amazon.S3.Model.GetObjectRequest()
{
BucketName = awsConfig.BucketName,
Key = ossRelativePath,
};
await (await amazonS3Client.GetObjectAsync(getObjectArgs)).WriteResponseStreamToFileAsync(localFilePath, true, CancellationToken.None);
var getObjectArgs = new GetObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithFile(localFilePath);
await minioClient.GetObjectAsync(getObjectArgs);
}
else
{
@ -497,7 +532,7 @@ namespace IRaCIS.Core.Application.Helper
do
{
// 使用 prefix 模拟目录结构,设置 MaxKeys 和 NextMarker
objectListing = _ossClient.ListObjects(new ListObjectsRequest(aliConfig.BucketName)
objectListing = _ossClient.ListObjects(new Aliyun.OSS.ListObjectsRequest(aliConfig.BucketName)
{
Prefix = prefix,
MaxKeys = 1000,
@ -509,7 +544,7 @@ namespace IRaCIS.Core.Application.Helper
// 删除获取到的文件
if (keys.Count > 0)
{
_ossClient.DeleteObjects(new DeleteObjectsRequest(aliConfig.BucketName, keys, false));
_ossClient.DeleteObjects(new Aliyun.OSS.DeleteObjectsRequest(aliConfig.BucketName, keys, false));
}
// 设置 NextMarker 以获取下一页的数据
@ -538,7 +573,7 @@ namespace IRaCIS.Core.Application.Helper
var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList();
var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList();
if (objects.Count > 0)
{
@ -555,7 +590,52 @@ namespace IRaCIS.Core.Application.Helper
}
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{
throw new BusinessValidationFailedException("未定义的存储介质类型");
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
//提供awsEndPoint域名进行访问配置
var clientConfig = new AmazonS3Config
{
ServiceURL = awsConfig.EndPoint
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
try
{
ListObjectsResponse objectListing = null;
string nextMarker = null;
do
{
// 使用 prefix 模拟目录结构,设置 MaxKeys 和 NextMarker
objectListing = await amazonS3Client.ListObjectsAsync(new Amazon.S3.Model.ListObjectsRequest()
{
Prefix = prefix,
MaxKeys = 1000,
Marker = nextMarker
});
//List<string> keys = objectListing.CommonPrefixes.Select(t => t.Key).ToList();
//// 删除获取到的文件
//if (keys.Count > 0)
//{
// //await amazonS3Client.DeleteObjectsAsync(new Amazon.S3.Model.DeleteObjectsRequest() { BucketName=awsConfig.BucketName,k}awsConfig.BucketName, keys, false));
//}
// 设置 NextMarker 以获取下一页的数据
nextMarker = objectListing.NextMarker;
} while (objectListing.IsTruncated);
}
catch (Exception ex)
{
Console.WriteLine($"Error: {ex.Message}");
}
}
else
{

View File

@ -59,6 +59,7 @@
<ItemGroup>
<PackageReference Include="Aliyun.OSS.SDK.NetCore" Version="2.14.1" />
<PackageReference Include="AWSSDK.S3" Version="3.7.402.4" />
<PackageReference Include="DocX" Version="3.0.1" />
<PackageReference Include="FreeSpire.Doc" Version="12.2.0" />
<PackageReference Include="Hangfire.Core" Version="1.8.14" />