From f4426dd2800d9f24b41abf4ea1108b1ea63c91a7 Mon Sep 17 00:00:00 2001 From: hang <872297557@qq.com> Date: Wed, 4 Sep 2024 11:04:39 +0800 Subject: [PATCH 1/2] =?UTF-8?q?s3=20=E5=A2=9E=E5=8A=A0region?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controllers/ExtraController.cs | 1 + IRaCIS.Core.API/appsettings.Test_IRC.json | 1 + IRaCIS.Core.Application/Helper/OSSService.cs | 158 +++++++++++++----- .../IRaCIS.Core.Application.csproj | 1 + 4 files changed, 122 insertions(+), 39 deletions(-) diff --git a/IRaCIS.Core.API/Controllers/ExtraController.cs b/IRaCIS.Core.API/Controllers/ExtraController.cs index b507b0813..e750b4727 100644 --- a/IRaCIS.Core.API/Controllers/ExtraController.cs +++ b/IRaCIS.Core.API/Controllers/ExtraController.cs @@ -380,6 +380,7 @@ namespace IRaCIS.Api.Controllers SessionToken= credentials.SessionToken, Expiration=credentials.Expiration, + BucketName = awsOptions.BucketName, EndPoint = awsOptions.EndPoint, ViewEndpoint = awsOptions.ViewEndpoint, diff --git a/IRaCIS.Core.API/appsettings.Test_IRC.json b/IRaCIS.Core.API/appsettings.Test_IRC.json index 3138ca442..078086d1d 100644 --- a/IRaCIS.Core.API/appsettings.Test_IRC.json +++ b/IRaCIS.Core.API/appsettings.Test_IRC.json @@ -37,6 +37,7 @@ "ViewEndpoint": "https://hir-oss.test.extimaging.com/irc-test" }, "AWS": { + "Region": "us-east-1", "EndPoint": "s3.us-east-1.amazonaws.com", "UseSSL": true, "RoleArn": "arn:aws:iam::471112624751:role/sts_s3_upload", diff --git a/IRaCIS.Core.Application/Helper/OSSService.cs b/IRaCIS.Core.Application/Helper/OSSService.cs index 2c28d3be2..01a6678f1 100644 --- a/IRaCIS.Core.Application/Helper/OSSService.cs +++ b/IRaCIS.Core.Application/Helper/OSSService.cs @@ -20,9 +20,15 @@ using System.Reactive.Linq; using IRaCIS.Core.Domain.Share; using static MassTransit.ValidationResultExtensions; using IRaCIS.Core.Infrastructure.NewtonsoftJson; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using MassTransit.Caching.Internals; namespace IRaCIS.Core.Application.Helper { + #region 绑定和返回模型 + [LowerCamelCaseJson] public class MinIOOptions : AWSOptions { @@ -30,7 +36,7 @@ namespace IRaCIS.Core.Application.Helper } - + public class AWSOptions { public string EndPoint { get; set; } @@ -41,6 +47,7 @@ namespace IRaCIS.Core.Application.Helper public string BucketName { get; set; } public string ViewEndpoint { get; set; } public int DurationSeconds { get; set; } + public string Region { get; set; } } public class AliyunOSSOptions @@ -118,6 +125,7 @@ namespace IRaCIS.Core.Application.Helper [LowerCamelCaseJson] public class AWSTempToken { + public string Region { get; set; } public string SessionToken { get; set; } public string EndPoint { get; set; } public string AccessKeyId { get; set; } @@ -134,6 +142,10 @@ namespace IRaCIS.Core.Application.Helper AWS = 2, } + #endregion + + // aws 参考链接 https://github.com/awsdocs/aws-doc-sdk-examples/tree/main/dotnetv3/S3/S3_Basics + public interface IOSSService { public Task UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true); @@ -170,8 +182,6 @@ namespace IRaCIS.Core.Application.Helper public async Task UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true) { var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}" : $"{oosFolderPath}/{fileRealName}"; - //var ossRelativePath = $"{oosFolderPath}/{Guid.NewGuid()}_{fileRealName}"; - //var ossRelativePath = oosFolderPath + "/" + fileRealName; try { @@ -215,20 +225,27 @@ namespace IRaCIS.Core.Application.Helper } else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") { - var minIOConfig = ObjectStoreServiceOptions.AWS; + var awsConfig = ObjectStoreServiceOptions.AWS; + // 提供awsAccessKeyId和awsSecretAccessKey构造凭证 + var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey); - var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}") - .WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL) - .Build(); + //提供awsEndPoint（域名）进行访问配置 + var clientConfig = new AmazonS3Config + { + ServiceURL = awsConfig.EndPoint + }; - var putObjectArgs = new PutObjectArgs() - .WithBucket(minIOConfig.BucketName) - .WithObject(ossRelativePath) - .WithStreamData(memoryStream) - .WithObjectSize(memoryStream.Length); + var amazonS3Client = new AmazonS3Client(credentials, clientConfig); - await minioClient.PutObjectAsync(putObjectArgs); + var putObjectRequest = new Amazon.S3.Model.PutObjectRequest() + { + BucketName = awsConfig.BucketName, + InputStream = memoryStream, + Key = ossRelativePath, + }; + + await amazonS3Client.PutObjectAsync(putObjectRequest); } else { @@ -256,6 +273,7 @@ namespace IRaCIS.Core.Application.Helper /// /// /// + /// /// /// public async Task UploadToOSSAsync(string localFilePath, string oosFolderPath, bool isFileNameAddGuid = true) @@ -265,9 +283,6 @@ namespace IRaCIS.Core.Application.Helper var ossRelativePath = isFileNameAddGuid ? $"{oosFolderPath}/{Guid.NewGuid()}_{localFileName}" : $"{oosFolderPath}/{localFileName}"; - //var ossRelativePath = oosFolderPath + "/" + localFileName; - - if (ObjectStoreServiceOptions.ObjectStoreUse == "AliyunOSS") { var aliConfig = ObjectStoreServiceOptions.AliyunOSS; @@ -296,19 +311,28 @@ namespace IRaCIS.Core.Application.Helper } else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") { - var minIOConfig = ObjectStoreServiceOptions.AWS; + var awsConfig = ObjectStoreServiceOptions.AWS; + // 提供awsAccessKeyId和awsSecretAccessKey构造凭证 + var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey); - var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}") - .WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL) - .Build(); + //提供awsEndPoint（域名）进行访问配置 + var clientConfig = new AmazonS3Config + { + ServiceURL = awsConfig.EndPoint + }; - var putObjectArgs = new PutObjectArgs() - .WithBucket(minIOConfig.BucketName) - .WithObject(ossRelativePath) - .WithFileName(localFilePath); + var amazonS3Client = new AmazonS3Client(credentials, clientConfig); + + var putObjectRequest = new Amazon.S3.Model.PutObjectRequest() + { + BucketName = awsConfig.BucketName, + FilePath = localFilePath, + Key = ossRelativePath, + }; + + await amazonS3Client.PutObjectAsync(putObjectRequest); - await minioClient.PutObjectAsync(putObjectArgs); } else { @@ -321,7 +345,7 @@ namespace IRaCIS.Core.Application.Helper public async Task DownLoadFromOSSAsync(string ossRelativePath, string localFilePath) { - + ossRelativePath = ossRelativePath.TrimStart('/'); try { @@ -362,18 +386,29 @@ namespace IRaCIS.Core.Application.Helper } else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") { - var minIOConfig = ObjectStoreServiceOptions.AWS; + var awsConfig = ObjectStoreServiceOptions.AWS; - var minioClient = new MinioClient().WithEndpoint($"{minIOConfig.EndPoint}") - .WithCredentials(minIOConfig.AccessKeyId, minIOConfig.SecretAccessKey).WithSSL(minIOConfig.UseSSL) - .Build(); + // 提供awsAccessKeyId和awsSecretAccessKey构造凭证 + var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey); + + //提供awsEndPoint（域名）进行访问配置 + var clientConfig = new AmazonS3Config + { + ServiceURL = awsConfig.EndPoint + }; + + var amazonS3Client = new AmazonS3Client(credentials, clientConfig); + + var getObjectArgs = new Amazon.S3.Model.GetObjectRequest() + { + BucketName = awsConfig.BucketName, + Key = ossRelativePath, + }; + + + await (await amazonS3Client.GetObjectAsync(getObjectArgs)).WriteResponseStreamToFileAsync(localFilePath, true, CancellationToken.None); - var getObjectArgs = new GetObjectArgs() - .WithBucket(minIOConfig.BucketName) - .WithObject(ossRelativePath) - .WithFile(localFilePath); - await minioClient.GetObjectAsync(getObjectArgs); } else { @@ -497,7 +532,7 @@ namespace IRaCIS.Core.Application.Helper do { // 使用 prefix 模拟目录结构，设置 MaxKeys 和 NextMarker - objectListing = _ossClient.ListObjects(new ListObjectsRequest(aliConfig.BucketName) + objectListing = _ossClient.ListObjects(new Aliyun.OSS.ListObjectsRequest(aliConfig.BucketName) { Prefix = prefix, MaxKeys = 1000, @@ -509,7 +544,7 @@ namespace IRaCIS.Core.Application.Helper // 删除获取到的文件 if (keys.Count > 0) { - _ossClient.DeleteObjects(new DeleteObjectsRequest(aliConfig.BucketName, keys, false)); + _ossClient.DeleteObjects(new Aliyun.OSS.DeleteObjectsRequest(aliConfig.BucketName, keys, false)); } // 设置 NextMarker 以获取下一页的数据 @@ -538,7 +573,7 @@ namespace IRaCIS.Core.Application.Helper - var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList(); + var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList(); if (objects.Count > 0) { @@ -555,7 +590,52 @@ namespace IRaCIS.Core.Application.Helper } else if (ObjectStoreServiceOptions.ObjectStoreUse == "AWS") { - throw new BusinessValidationFailedException("未定义的存储介质类型"); + + var awsConfig = ObjectStoreServiceOptions.AWS; + + + // 提供awsAccessKeyId和awsSecretAccessKey构造凭证 + var credentials = new BasicAWSCredentials(awsConfig.AccessKeyId, awsConfig.SecretAccessKey); + + //提供awsEndPoint（域名）进行访问配置 + var clientConfig = new AmazonS3Config + { + ServiceURL = awsConfig.EndPoint + }; + + var amazonS3Client = new AmazonS3Client(credentials, clientConfig); + + try + { + ListObjectsResponse objectListing = null; + string nextMarker = null; + do + { + // 使用 prefix 模拟目录结构，设置 MaxKeys 和 NextMarker + objectListing = await amazonS3Client.ListObjectsAsync(new Amazon.S3.Model.ListObjectsRequest() + { + Prefix = prefix, + MaxKeys = 1000, + Marker = nextMarker + }); + + //List keys = objectListing.CommonPrefixes.Select(t => t.Key).ToList(); + + //// 删除获取到的文件 + //if (keys.Count > 0) + //{ + // //await amazonS3Client.DeleteObjectsAsync(new Amazon.S3.Model.DeleteObjectsRequest() { BucketName=awsConfig.BucketName,k}awsConfig.BucketName, keys, false)); + //} + + // 设置 NextMarker 以获取下一页的数据 + nextMarker = objectListing.NextMarker; + + } while (objectListing.IsTruncated); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + } } else { diff --git a/IRaCIS.Core.Application/IRaCIS.Core.Application.csproj b/IRaCIS.Core.Application/IRaCIS.Core.Application.csproj index daa98ea98..c1eb5c0a7 100644 --- a/IRaCIS.Core.Application/IRaCIS.Core.Application.csproj +++ b/IRaCIS.Core.Application/IRaCIS.Core.Application.csproj @@ -59,6 +59,7 @@ + From 6cc4c32625293bf9e9f6768d969a6b304cd78b93 Mon Sep 17 00:00:00 2001 From: hang <872297557@qq.com> Date: Wed, 4 Sep 2024 11:14:52 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9s3=20minio=20=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controllers/ExtraController.cs | 4 +- IRaCIS.Core.Application/Helper/OSSService.cs | 61 +++++++++++-------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/IRaCIS.Core.API/Controllers/ExtraController.cs b/IRaCIS.Core.API/Controllers/ExtraController.cs index e750b4727..baa34b64f 100644 --- a/IRaCIS.Core.API/Controllers/ExtraController.cs +++ b/IRaCIS.Core.API/Controllers/ExtraController.cs @@ -379,7 +379,7 @@ namespace IRaCIS.Api.Controllers SecretAccessKey = credentials.SecretAccessKey, SessionToken= credentials.SessionToken, Expiration=credentials.Expiration, - + Region=awsOptions.Region, BucketName = awsOptions.BucketName, EndPoint = awsOptions.EndPoint, @@ -422,7 +422,7 @@ namespace IRaCIS.Api.Controllers SecretAccessKey = credentials.SecretAccessKey, SessionToken = credentials.SessionToken, Expiration = credentials.Expiration, - + Region = awsOptions.Region, BucketName = awsOptions.BucketName, EndPoint = awsOptions.EndPoint, ViewEndpoint = awsOptions.ViewEndpoint, diff --git a/IRaCIS.Core.Application/Helper/OSSService.cs b/IRaCIS.Core.Application/Helper/OSSService.cs index 01a6678f1..061150189 100644 --- a/IRaCIS.Core.Application/Helper/OSSService.cs +++ b/IRaCIS.Core.Application/Helper/OSSService.cs @@ -573,7 +573,15 @@ namespace IRaCIS.Core.Application.Helper - var objects = minioClient.ListObjectsAsync(listArgs).ToListObservable().Select(t => t.Key).ToList(); + // 创建一个空列表用于存储对象键 + var objects = new List(); + + // 使用 await foreach 来异步迭代对象列表 + await foreach (var item in minioClient.ListObjectsEnumAsync(listArgs)) + { + objects.Add(item.Key); + } + if (objects.Count > 0) { @@ -605,37 +613,38 @@ namespace IRaCIS.Core.Application.Helper var amazonS3Client = new AmazonS3Client(credentials, clientConfig); - try + // 列出指定前缀下的所有对象 + var listObjectsRequest = new ListObjectsV2Request { - ListObjectsResponse objectListing = null; - string nextMarker = null; - do + BucketName = awsConfig.BucketName, + Prefix = prefix + }; + + var listObjectsResponse = await amazonS3Client.ListObjectsV2Async(listObjectsRequest); + + if (listObjectsResponse.S3Objects.Count > 0) + { + // 准备删除请求 + var deleteObjectsRequest = new Amazon.S3.Model.DeleteObjectsRequest { - // 使用 prefix 模拟目录结构，设置 MaxKeys 和 NextMarker - objectListing = await amazonS3Client.ListObjectsAsync(new Amazon.S3.Model.ListObjectsRequest() + BucketName = awsConfig.BucketName, + Objects = new List() + }; + + foreach (var s3Object in listObjectsResponse.S3Objects) + { + deleteObjectsRequest.Objects.Add(new KeyVersion { - Prefix = prefix, - MaxKeys = 1000, - Marker = nextMarker + Key = s3Object.Key }); + } - //List keys = objectListing.CommonPrefixes.Select(t => t.Key).ToList(); - - //// 删除获取到的文件 - //if (keys.Count > 0) - //{ - // //await amazonS3Client.DeleteObjectsAsync(new Amazon.S3.Model.DeleteObjectsRequest() { BucketName=awsConfig.BucketName,k}awsConfig.BucketName, keys, false)); - //} - - // 设置 NextMarker 以获取下一页的数据 - nextMarker = objectListing.NextMarker; - - } while (objectListing.IsTruncated); - } - catch (Exception ex) - { - Console.WriteLine($"Error: {ex.Message}"); + // 批量删除对象 + var deleteObjectsResponse = await amazonS3Client.DeleteObjectsAsync(deleteObjectsRequest); } + + + } else {