XCKJ
/
irc-netcore-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s3 增加region

IRC_NewDev
hang 2024-09-04 11:04:39 +08:00
parent cec6174bb6
commit f4426dd280
4 changed files with 122 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
IRaCIS.Core.API/Controllers/ExtraController.cs
View File

@ -380,6 +380,7 @@ namespace IRaCIS.Api.Controllers
SessionToken= credentials.SessionToken, SessionToken= credentials.SessionToken,
Expiration=credentials.Expiration, Expiration=credentials.Expiration,
BucketName = awsOptions.BucketName, BucketName = awsOptions.BucketName,
EndPoint = awsOptions.EndPoint, EndPoint = awsOptions.EndPoint,
ViewEndpoint = awsOptions.ViewEndpoint, ViewEndpoint = awsOptions.ViewEndpoint,

1
IRaCIS.Core.API/appsettings.Test_IRC.json
View File

@ -37,6 +37,7 @@
"ViewEndpoint": "https://hir-oss.test.extimaging.com/irc-test" "ViewEndpoint": "https://hir-oss.test.extimaging.com/irc-test"
}, },
"AWS": { "AWS": {
"Region": "us-east-1",
"EndPoint": "s3.us-east-1.amazonaws.com", "EndPoint": "s3.us-east-1.amazonaws.com",
"UseSSL": true, "UseSSL": true,
"RoleArn": "arn:aws:iam::471112624751:role/sts_s3_upload", "RoleArn": "arn:aws:iam::471112624751:role/sts_s3_upload",

158
IRaCIS.Core.Application/Helper/OSSService.cs
View File

@ -20,9 +20,15 @@ using System.Reactive.Linq;
using IRaCIS.Core.Domain.Share; using IRaCIS.Core.Domain.Share;
using static MassTransit.ValidationResultExtensions; using static MassTransit.ValidationResultExtensions;
using IRaCIS.Core.Infrastructure.NewtonsoftJson; using IRaCIS.Core.Infrastructure.NewtonsoftJson;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using MassTransit.Caching.Internals;
namespace IRaCIS.Core.Application.Helper namespace IRaCIS.Core.Application.Helper
{ {
#region 绑定和返回模型
[LowerCamelCaseJson] [LowerCamelCaseJson]
public class MinIOOptions : AWSOptions public class MinIOOptions : AWSOptions
{ {
@ -30,7 +36,7 @@ namespace IRaCIS.Core.Application.Helper
} }
public class AWSOptions public class AWSOptions
{ {
public string EndPoint { get; set; } public string EndPoint { get; set; }
@ -41,6 +47,7 @@ namespace IRaCIS.Core.Application.Helper
public string BucketName { get; set; } public string BucketName { get; set; }
public string ViewEndpoint { get; set; } public string ViewEndpoint { get; set; }
public int DurationSeconds { get; set; } public int DurationSeconds { get; set; }
public string Region { get; set; }
} }
public class AliyunOSSOptions public class AliyunOSSOptions
@ -118,6 +125,7 @@ namespace IRaCIS.Core.Application.Helper
[LowerCamelCaseJson] [LowerCamelCaseJson]
public class AWSTempToken public class AWSTempToken
{ {
public string Region { get; set; }
public string SessionToken { get; set; } public string SessionToken { get; set; }
public string EndPoint { get; set; } public string EndPoint { get; set; }
public string AccessKeyId { get; set; } public string AccessKeyId { get; set; }
@ -134,6 +142,10 @@ namespace IRaCIS.Core.Application.Helper
AWS = 2, AWS = 2,
} }
#endregion
// aws 参考链接 https://github.com/awsdocs/aws-doc-sdk-examples/tree/main/dotnetv3/S3/S3_Basics
public interface IOSSService public interface IOSSService
{ {
public Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true); public Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true);
@ -170,8 +182,6 @@ namespace IRaCIS.Core.Application.Helper
public async Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true) public async Task<string> UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true)
{ {
var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}" : $"{oosFolderPath}/{fileRealName}"; var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}" : $"{oosFolderPath}/{fileRealName}";
//var ossRelativePath = $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}";
//var ossRelativePath = oosFolderPath + "/" + fileRealName;
try try
{ {
@ -215,20 +225,27 @@ namespace IRaCIS.Core.Application.Helper
} }
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{ {
var minIOConfig = ObjectStoreServiceOptions.AWS; var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}") //提供awsEndPoint（域名）进行访问配置
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL) var clientConfig = new AmazonS3Config
.Build(); {
ServiceURL = awsConfig.EndPoint
};
var putObjectArgs = new PutObjectArgs() var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithStreamData(memoryStream)
.WithObjectSize(memoryStream.Length);
await minioClient.PutObjectAsync(putObjectArgs); var putObjectRequest = new Amazon.S3.Model.PutObjectRequest()
{
BucketName = awsConfig.BucketName,
InputStream = memoryStream,
Key = ossRelativePath,
};
await amazonS3Client.PutObjectAsync(putObjectRequest);
} }
else else
{ {
@ -256,6 +273,7 @@ namespace IRaCIS.Core.Application.Helper
/// </summary> /// </summary>
/// <param name="localFilePath"></param> /// <param name="localFilePath"></param>
/// <param name="oosFolderPath"></param> /// <param name="oosFolderPath"></param>
/// <param name="isFileNameAddGuid"></param>
/// <returns></returns> /// <returns></returns>
/// <exception cref="BusinessValidationFailedException"></exception> /// <exception cref="BusinessValidationFailedException"></exception>
public async Task<string> UploadToOSSAsync(string localFilePath, string oosFolderPath, bool isFileNameAddGuid = true) public async Task<string> UploadToOSSAsync(string localFilePath, string oosFolderPath, bool isFileNameAddGuid = true)
@ -265,9 +283,6 @@ namespace IRaCIS.Core.Application.Helper
var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{localFileName}" : $"{oosFolderPath}/{localFileName}"; var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{localFileName}" : $"{oosFolderPath}/{localFileName}";
//var ossRelativePath = oosFolderPath + "/" + localFileName;
if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS") if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS")
{ {
var aliConfig = ObjectStoreServiceOptions.AliyunOSS; var aliConfig = ObjectStoreServiceOptions.AliyunOSS;
@ -296,19 +311,28 @@ namespace IRaCIS.Core.Application.Helper
} }
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{ {
var minIOConfig = ObjectStoreServiceOptions.AWS; var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}") //提供awsEndPoint（域名）进行访问配置
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL) var clientConfig = new AmazonS3Config
.Build(); {
ServiceURL = awsConfig.EndPoint
};
var putObjectArgs = new PutObjectArgs() var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath) var putObjectRequest = new Amazon.S3.Model.PutObjectRequest()
.WithFileName(localFilePath); {
BucketName = awsConfig.BucketName,
FilePath = localFilePath,
Key = ossRelativePath,
};
await amazonS3Client.PutObjectAsync(putObjectRequest);
await minioClient.PutObjectAsync(putObjectArgs);
} }
else else
{ {
@ -321,7 +345,7 @@ namespace IRaCIS.Core.Application.Helper
public async Task DownLoadFromOSSAsync(string ossRelativePath, string localFilePath) public async Task DownLoadFromOSSAsync(string ossRelativePath, string localFilePath)
{ {
ossRelativePath = ossRelativePath.TrimStart('/'); ossRelativePath = ossRelativePath.TrimStart('/');
try try
{ {
@ -362,18 +386,29 @@ namespace IRaCIS.Core.Application.Helper
} }
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{ {
var minIOConfig = ObjectStoreServiceOptions.AWS; var awsConfig = ObjectStoreServiceOptions.AWS;
var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}") // 提供awsAccessKeyId和awsSecretAccessKey构造凭证
.WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL) var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
.Build();
//提供awsEndPoint（域名）进行访问配置
var clientConfig = new AmazonS3Config
{
ServiceURL = awsConfig.EndPoint
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
var getObjectArgs = new Amazon.S3.Model.GetObjectRequest()
{
BucketName = awsConfig.BucketName,
Key = ossRelativePath,
};
await (await amazonS3Client.GetObjectAsync(getObjectArgs)).WriteResponseStreamToFileAsync(localFilePath, true, CancellationToken.None);
var getObjectArgs = new GetObjectArgs()
.WithBucket(minIOConfig.BucketName)
.WithObject(ossRelativePath)
.WithFile(localFilePath);
await minioClient.GetObjectAsync(getObjectArgs);
} }
else else
{ {
@ -497,7 +532,7 @@ namespace IRaCIS.Core.Application.Helper
do do
{ {
// 使用 prefix 模拟目录结构，设置 MaxKeys 和 NextMarker // 使用 prefix 模拟目录结构，设置 MaxKeys 和 NextMarker
objectListing = _ossClient.ListObjects(new ListObjectsRequest(aliConfig.BucketName) objectListing = _ossClient.ListObjects(new Aliyun.OSS.ListObjectsRequest(aliConfig.BucketName)
{ {
Prefix = prefix, Prefix = prefix,
MaxKeys = 1000, MaxKeys = 1000,
@ -509,7 +544,7 @@ namespace IRaCIS.Core.Application.Helper
// 删除获取到的文件 // 删除获取到的文件
if (keys.Count > 0) if (keys.Count > 0)
{ {
_ossClient.DeleteObjects(new DeleteObjectsRequest(aliConfig.BucketName, keys, false)); _ossClient.DeleteObjects(new Aliyun.OSS.DeleteObjectsRequest(aliConfig.BucketName, keys, false));
} }
// 设置 NextMarker 以获取下一页的数据 // 设置 NextMarker 以获取下一页的数据
@ -538,7 +573,7 @@ namespace IRaCIS.Core.Application.Helper
var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList(); var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList();
if (objects.Count > 0) if (objects.Count > 0)
{ {
@ -555,7 +590,52 @@ namespace IRaCIS.Core.Application.Helper
} }
else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS")
{ {
throw new BusinessValidationFailedException("未定义的存储介质类型");
var awsConfig = ObjectStoreServiceOptions.AWS;
// 提供awsAccessKeyId和awsSecretAccessKey构造凭证
var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey);
//提供awsEndPoint（域名）进行访问配置
var clientConfig = new AmazonS3Config
{
ServiceURL = awsConfig.EndPoint
};
var amazonS3Client = new AmazonS3Client(credentials, clientConfig);
try
{
ListObjectsResponse objectListing = null;
string nextMarker = null;
do
{
// 使用 prefix 模拟目录结构，设置 MaxKeys 和 NextMarker
objectListing = await amazonS3Client.ListObjectsAsync(new Amazon.S3.Model.ListObjectsRequest()
{
Prefix = prefix,
MaxKeys = 1000,
Marker = nextMarker
});
//List<string> keys = objectListing.CommonPrefixes.Select(t => t.Key).ToList();
//// 删除获取到的文件
//if (keys.Count > 0)
//{
// //await amazonS3Client.DeleteObjectsAsync(new Amazon.S3.Model.DeleteObjectsRequest() { BucketName=awsConfig.BucketName,k}awsConfig.BucketName, keys, false));
//}
// 设置 NextMarker 以获取下一页的数据
nextMarker = objectListing.NextMarker;
} while (objectListing.IsTruncated);
}
catch (Exception ex)
{
Console.WriteLine($"Error: {ex.Message}");
}
} }
else else
{ {

1
IRaCIS.Core.Application/IRaCIS.Core.Application.csproj
View File

@ -59,6 +59,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Aliyun.OSS.SDK.NetCore" Version="2.14.1" /> <PackageReference Include="Aliyun.OSS.SDK.NetCore" Version="2.14.1" />
<PackageReference Include="AWSSDK.S3" Version="3.7.402.4" />
<PackageReference Include="DocX" Version="3.0.1" /> <PackageReference Include="DocX" Version="3.0.1" />
<PackageReference Include="FreeSpire.Doc" Version="12.2.0" /> <PackageReference Include="FreeSpire.Doc" Version="12.2.0" />
<PackageReference Include="Hangfire.Core" Version="1.8.14" /> <PackageReference Include="Hangfire.Core" Version="1.8.14" />