From fbd128138d1cb2e029d135756b974d0f7c3e6bb9 Mon Sep 17 00:00:00 2001 From: hang <872297557@qq.com> Date: Thu, 8 Jan 2026 10:09:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BD=B1=E5=83=8F=E6=81=A2=E5=A4=8D=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- IRaCIS.Core.Application/Helper/OSSService.cs | 249 +++++++----------- .../IRaCIS.Core.Application.xml | 2 +- IRaCIS.Core.Application/TestService.cs | 4 +- 3 files changed, 99 insertions(+), 156 deletions(-) diff --git a/IRaCIS.Core.Application/Helper/OSSService.cs b/IRaCIS.Core.Application/Helper/OSSService.cs index b272d5a82..2339f8df6 100644 --- a/IRaCIS.Core.Application/Helper/OSSService.cs +++ b/IRaCIS.Core.Application/Helper/OSSService.cs @@ -144,7 +144,7 @@ public enum ObjectStoreUse public interface IOSSService { public Task SetImmediateArchiveRule(string prefix, string ruleId = "immediate-archive", bool isDelete = false); - + public Task RestoreFilesByPrefixAsync(string prefix, int restoreDays = 3, int batchSize = 100); public Task UploadToOSSAsync(Stream fileStream, string oosFolderPath, string fileRealName, bool isFileNameAddGuid = true); @@ -384,10 +384,7 @@ public class OSSService : IOSSService /// 解冻后文件保持可读的天数（默认3天） /// 解冻优先级（仅AWS有效） /// 批量处理大小（默认100） - public async Task RestoreFilesByPrefixAsync(string prefix, - int restoreDays = 3, - string restoreTier = "Standard", - int batchSize = 100) + public async Task RestoreFilesByPrefixAsync(string prefix, int restoreDays = 3, int batchSize = 100) { BackBatchGetToken(); @@ -410,6 +407,8 @@ public class OSSService : IOSSService { Console.WriteLine($"开始解冻阿里云OSS文件，前缀: {prefix}"); + var allObjects = new List(); + // 1. 分页列举文件 string nextMarker = null; ObjectListing result = null; @@ -425,74 +424,56 @@ public class OSSService : IOSSService result = client.ListObjects(listRequest); - // 2. 批量处理每个文件 - var restoreTasks = new List(); + allObjects.AddRange(result.ObjectSummaries); - foreach (var obj in result.ObjectSummaries) - { - // 跳过非归档/冷归档文件 - if (obj.StorageClass != StorageClass.Archive.ToString()) - { - Console.WriteLine($"跳过非归档文件: {obj.Key} (类型: {obj.StorageClass})"); - totalSkipped++; - continue; - } - // 提交解冻任务 - var task = Task.Run(() => - { - try - { - var restoreRequest = new Aliyun.OSS.RestoreObjectRequest(bucketName, obj.Key) - { - Days = restoreDays - }; - - client.RestoreObject(restoreRequest); - - Interlocked.Increment(ref totalRestored); - Console.WriteLine($"✅ 已提交解冻: {obj.Key} (天数: {restoreDays})"); - } - catch (OssException ex) - { - if (ex.ErrorCode == "RestoreAlreadyInProgress") - { - Console.WriteLine($"⚠️ 解冻已在进行中: {obj.Key}"); - } - else - { - Interlocked.Increment(ref totalFailed); - Console.WriteLine($"❌ 解冻失败 [{obj.Key}]: {ex.Message}"); - } - } - catch (Exception ex) - { - Interlocked.Increment(ref totalFailed); - Console.WriteLine($"❌ 解冻失败 [{obj.Key}]: {ex.Message}"); - } - }); - - restoreTasks.Add(task); - - // 控制并发，避免请求过快 - if (restoreTasks.Count >= 10) // 并发10个任务 - { - await Task.WhenAll(restoreTasks); - restoreTasks.Clear(); - await Task.Delay(100); // 短暂延迟 - } - } - - // 等待剩余任务完成 - if (restoreTasks.Count > 0) - { - await Task.WhenAll(restoreTasks); - } nextMarker = result.NextMarker; } while (result.IsTruncated); + // 2️⃣ 并行解冻（控制并发） + Parallel.ForEach( + allObjects, + new ParallelOptions + { + MaxDegreeOfParallelism = 5 // ⭐ 推荐 5~10 + }, + obj => + { + // 只处理归档 + if (obj.StorageClass != StorageClass.Archive.ToString()) + { + Interlocked.Increment(ref totalSkipped); + return; + } + + try + { + var restoreRequest = new Aliyun.OSS.RestoreObjectRequest(bucketName, obj.Key) + { + Days = restoreDays + }; + + client.RestoreObject(restoreRequest); + + Interlocked.Increment(ref totalRestored); + Console.WriteLine($"✅ 提交解冻: {obj.Key}"); + } + catch (OssException ex) when (ex.ErrorCode == "RestoreAlreadyInProgress") + { + // 已在解冻中，算成功 + Interlocked.Increment(ref totalSkipped); + Console.WriteLine($"⚠️ 已在解冻中: {obj.Key}"); + } + catch (Exception ex) + { + Interlocked.Increment(ref totalFailed); + Console.WriteLine($"❌ 解冻失败: {obj.Key} - {ex.Message}"); + } + } + ); + // 3. 输出统计结果 Console.WriteLine("\n================ 解冻完成 ================"); Console.WriteLine($"总计处理: {totalRestored + totalSkipped + totalFailed} 个文件"); @@ -540,6 +521,8 @@ public class OSSService : IOSSService { Console.WriteLine($"开始解冻AWS S3文件，前缀: {prefix}"); + var allObjects = new List(); + // 1. 分页列举文件 string continuationToken = null; ListObjectsV2Response response = null; @@ -556,100 +539,58 @@ public class OSSService : IOSSService response = await client.ListObjectsV2Async(listRequest); - // 2. 批量处理每个文件 - var restoreTasks = new List(); + allObjects.AddRange(response.S3Objects); - foreach (var obj in response.S3Objects) - { - // 需要先获取对象元数据以确定存储类型 - var metadataRequest = new Amazon.S3.Model.GetObjectMetadataRequest - { - BucketName = bucketName, - Key = obj.Key - }; - - try - { - var metadata = await client.GetObjectMetadataAsync(metadataRequest); - - // 检查是否为归档类型 - bool isGlacierObject = metadata.StorageClass == S3StorageClass.Glacier; - - if (!isGlacierObject) - { - Console.WriteLine($"跳过非归档文件: {obj.Key} (类型: {metadata.StorageClass})"); - totalSkipped++; - continue; - } - - // 提交解冻任务 - var task = Task.Run(async () => - { - try - { - var restoreRequest = new Amazon.S3.Model.RestoreObjectRequest - { - BucketName = bucketName, - Key = obj.Key, - Days = restoreDays - }; - - await client.RestoreObjectAsync(restoreRequest); - - Interlocked.Increment(ref totalRestored); - Console.WriteLine($"✅ 已提交解冻: {obj.Key} (天数: {restoreDays}, 优先级: {restoreTier})"); - } - catch (AmazonS3Exception ex) - { - if (ex.ErrorCode == "RestoreAlreadyInProgress") - { - Console.WriteLine($"⚠️ 解冻已在进行中: {obj.Key}"); - } - else if (ex.ErrorCode == "AccessDenied") - { - Interlocked.Increment(ref totalFailed); - Console.WriteLine($"❌ 权限不足 [{obj.Key}]: 需要 s3:RestoreObject 权限"); - } - else - { - Interlocked.Increment(ref totalFailed); - Console.WriteLine($"❌ 解冻失败 [{obj.Key}]: {ex.Message}"); - } - } - catch (Exception ex) - { - Interlocked.Increment(ref totalFailed); - Console.WriteLine($"❌ 解冻失败 [{obj.Key}]: {ex.Message}"); - } - }); - - restoreTasks.Add(task); - - // 控制并发 - if (restoreTasks.Count >= 5) // AWS建议较低并发 - { - await Task.WhenAll(restoreTasks); - restoreTasks.Clear(); - await Task.Delay(200); // AWS有速率限制 - } - } - catch (Exception ex) - { - Console.WriteLine($"❌ 获取元数据失败 [{obj.Key}]: {ex.Message}"); - totalFailed++; - } - } - - // 等待剩余任务 - if (restoreTasks.Count > 0) - { - await Task.WhenAll(restoreTasks); - } continuationToken = response.NextContinuationToken; } while (response.IsTruncated == true); + // 2️⃣ 并行解冻（控制并发） + await Parallel.ForEachAsync( + allObjects, + new ParallelOptions + { + MaxDegreeOfParallelism = 5 // ⭐ 推荐 5~10 + }, + async (obj, ct) => + { + // 只处理归档 + if (obj.StorageClass != S3StorageClass.Glacier) + { + Interlocked.Increment(ref totalSkipped); + return; + } + + try + { + var restoreRequest = new Amazon.S3.Model.RestoreObjectRequest + { + BucketName = bucketName, + Key = obj.Key, + Days = restoreDays, + + }; + + await client.RestoreObjectAsync(restoreRequest); + + Interlocked.Increment(ref totalRestored); + Console.WriteLine($"✅ 提交解冻: {obj.Key}"); + } + catch (OssException ex) when (ex.ErrorCode == "RestoreAlreadyInProgress") + { + // 已在解冻中，算成功 + Interlocked.Increment(ref totalSkipped); + Console.WriteLine($"⚠️ 已在解冻中: {obj.Key}"); + } + catch (Exception ex) + { + Interlocked.Increment(ref totalFailed); + Console.WriteLine($"❌ 解冻失败: {obj.Key} - {ex.Message}"); + } + } + ); + // 3. 输出统计结果 Console.WriteLine("\n================ 解冻完成 ================"); Console.WriteLine($"总计处理: {totalRestored + totalSkipped + totalFailed} 个文件"); diff --git a/IRaCIS.Core.Application/IRaCIS.Core.Application.xml b/IRaCIS.Core.Application/IRaCIS.Core.Application.xml index a790082c6..c088abee4 100644 --- a/IRaCIS.Core.Application/IRaCIS.Core.Application.xml +++ b/IRaCIS.Core.Application/IRaCIS.Core.Application.xml @@ -15748,7 +15748,7 @@ 规则ID，默认为"immediate-archive" 默认是添加/更新 - + 坑方法，会清空之前的规则 diff --git a/IRaCIS.Core.Application/TestService.cs b/IRaCIS.Core.Application/TestService.cs index be1248356..c98bf2044 100644 --- a/IRaCIS.Core.Application/TestService.cs +++ b/IRaCIS.Core.Application/TestService.cs @@ -80,7 +80,9 @@ namespace IRaCIS.Core.Application.Service { if (storageClass == StorageClass.IA || storageClass == StorageClass.Archive || storageClass == StorageClass.ColdArchive || storageClass == StorageClass.DeepColdArchive) { - _IOSSService.SetImmediateArchiveRule($"Test-Archive/Archive{(int)storageClass}/"); + //await _IOSSService.SetImmediateArchiveRule($"Test-Archive/Archive{(int)storageClass}/"); + + await _IOSSService.RestoreFilesByPrefixAsync($"Test-Archive/Archive{(int)storageClass}/"); }