增加内存优先级队列、后端上传的地方都设置类别,和业务参数、增加启动后台恢复队列任务
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
using IRaCIS.Core.Application.Helper;
|
||||
using IRaCIS.Core.Application.Service;
|
||||
using IRaCIS.Core.Domain.Models;
|
||||
using IRaCIS.Core.Infra.EFCore;
|
||||
using IRaCIS.Core.SCP.Service;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace IRaCIS.Core.SCP;
|
||||
|
||||
|
||||
public class SyncFileRecoveryService(IServiceScopeFactory _scopeFactory, FileSyncQueue _fileSyncQueue) : BackgroundService
|
||||
{
|
||||
|
||||
private readonly int _pageSize = 500;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var fileUploadRecordRepository = scope.ServiceProvider.GetRequiredService<IRepository<FileUploadRecord>>();
|
||||
|
||||
// 延迟启动,保证主机快速启动
|
||||
await Task.Delay(5000, stoppingToken);
|
||||
|
||||
int page = 0;
|
||||
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// 分页获取未入队任务
|
||||
var pending = await fileUploadRecordRepository
|
||||
.Where(x => x.IsNeedSync == true && (x.IsSync == false || x.IsSync == null))
|
||||
.OrderByDescending(x => x.Priority)
|
||||
.Select(t => new { t.Id, t.Priority })
|
||||
.Skip(page * _pageSize)
|
||||
.Take(_pageSize)
|
||||
.ToListAsync(stoppingToken);
|
||||
|
||||
if (!pending.Any())
|
||||
break; // 扫描完毕,退出循环
|
||||
|
||||
foreach (var file in pending)
|
||||
{
|
||||
//file.IsQueued = true; // 避免重复入队
|
||||
_fileSyncQueue.Enqueue(file.Id, file.Priority ?? 0); // 放入队列
|
||||
}
|
||||
|
||||
page++; // 下一页
|
||||
await Task.Delay(200, stoppingToken); // 缓解数据库压力
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class FileSyncWorker(IServiceScopeFactory _scopeFactory, ILogger<FileSyncWorker> _logger, FileSyncQueue _fileSyncQueue) : BackgroundService
|
||||
{
|
||||
|
||||
// ⭐ 自动根据服务器CPU
|
||||
private readonly int _workerCount = Math.Max(1, Environment.ProcessorCount - 1);
|
||||
|
||||
|
||||
|
||||
protected override Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
for (int i = 0; i < _workerCount; i++)
|
||||
Task.Run(() => WorkerLoop(stoppingToken));
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
|
||||
private async Task WorkerLoop(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var id = await _fileSyncQueue.DequeueAsync(stoppingToken);
|
||||
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var _fileUploadRecordRepository = scope.ServiceProvider.GetRequiredService<IRepository<FileUploadRecord>>();
|
||||
var _uploadFileSyncRecordRepository = scope.ServiceProvider.GetRequiredService<IRepository<UploadFileSyncRecord>>();
|
||||
var oss = scope.ServiceProvider.GetRequiredService<IOSSService>();
|
||||
|
||||
var file = await _fileUploadRecordRepository.FirstOrDefaultAsync(t => t.Id == id);
|
||||
|
||||
if (file == null || file.IsNeedSync != true)
|
||||
return;
|
||||
|
||||
var log = new UploadFileSyncRecord
|
||||
{
|
||||
FileUploadRecordId = id,
|
||||
StartTime = DateTime.Now,
|
||||
JobState = jobState.RUNNING
|
||||
};
|
||||
|
||||
await _uploadFileSyncRecordRepository.AddAsync(log);
|
||||
await _uploadFileSyncRecordRepository.SaveChangesAsync(stoppingToken);
|
||||
|
||||
try
|
||||
{
|
||||
await oss.SyncFileAsync(file.Path.TrimStart("/"), file.UploadRegion == "CN" ? ObjectStoreUse.AliyunOSS : ObjectStoreUse.AWS, file.UploadRegion == "CN" ? ObjectStoreUse.AWS : ObjectStoreUse.AliyunOSS);
|
||||
|
||||
file.IsSync = true;
|
||||
file.SyncFinishedTime = DateTime.UtcNow;
|
||||
|
||||
log.JobState = jobState.SUCCESS;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
log.JobState = jobState.FAILED;
|
||||
|
||||
log.Msg = ex.Message[..300];
|
||||
}
|
||||
|
||||
log.EndTime = DateTime.UtcNow;
|
||||
|
||||
await _uploadFileSyncRecordRepository.SaveChangesAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Sync failed {Id}", id);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,6 +63,9 @@ builder.Host
|
||||
#region 配置服务
|
||||
var _configuration = builder.Configuration;
|
||||
|
||||
builder.Services.AddHostedService<SyncFileRecoveryService>();
|
||||
builder.Services.AddHostedService<FileSyncWorker>();
|
||||
|
||||
//健康检查
|
||||
builder.Services.AddHealthChecks();
|
||||
|
||||
|
||||
@@ -644,7 +644,7 @@ namespace IRaCIS.Core.SCP.Service
|
||||
ms.Position = 0;
|
||||
|
||||
//irc 从路径最后一截取Guid
|
||||
storeRelativePath = await ossService.UploadToOSSAsync(ms, ossFolderPath, instanceId.ToString(), false);
|
||||
storeRelativePath = await ossService.UploadToOSSAsync(ms, ossFolderPath, instanceId.ToString(), false, uploadInfo: new FileUploadRecordAddOrEdit() { TrialId = _trialId, BatchDataType = BatchDataType.PACSReceive });
|
||||
|
||||
fileSize = ms.Length;
|
||||
|
||||
@@ -674,7 +674,7 @@ namespace IRaCIS.Core.SCP.Service
|
||||
|
||||
// 上传缩略图到 OSS
|
||||
|
||||
var seriesPath = await ossService.UploadToOSSAsync(memoryStream, ossFolderPath, $"{seriesId.ToString()}_{instanceId.ToString()}.preview.jpg", false);
|
||||
var seriesPath = await ossService.UploadToOSSAsync(memoryStream, ossFolderPath, $"{seriesId.ToString()}_{instanceId.ToString()}.preview.jpg", false,uploadInfo: new FileUploadRecordAddOrEdit() { TrialId = _trialId, BatchDataType = BatchDataType.PACSReceive });
|
||||
|
||||
|
||||
series.ImageResizePath = seriesPath;
|
||||
|
||||
@@ -69,6 +69,8 @@ public static class CacheKeys
|
||||
public static string UserMFAVerifyPass(Guid userId, string browserFingerprint) => $"UserMFAVerifyPass:{userId}:{browserFingerprint}";
|
||||
|
||||
public static string TrialSiteInfo(Guid trialSiteId) => $"{trialSiteId}TrialSiteInfo";
|
||||
|
||||
public static string TrialDataStoreType(Guid trialId) => $"TrialDataStoreType:{trialId}";
|
||||
}
|
||||
|
||||
public static class CacheHelper
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
|
||||
//--------------------------------------------------------------------
|
||||
// 此代码由liquid模板自动生成 byzhouhang 20240909
|
||||
// 生成时间 2026-03-10 06:15:17Z
|
||||
// 对此文件的更改可能会导致不正确的行为,并且如果重新生成代码,这些更改将会丢失。
|
||||
//--------------------------------------------------------------------
|
||||
using AutoMapper;
|
||||
using IRaCIS.Core.Application.Helper;
|
||||
|
||||
using IRaCIS.Core.Domain.Models;
|
||||
using IRaCIS.Core.Domain.Share;
|
||||
using IRaCIS.Core.Infra.EFCore;
|
||||
using IRaCIS.Core.Infra.EFCore.Common;
|
||||
using IRaCIS.Core.Infrastructure.Extention;
|
||||
using IRaCIS.Core.SCP;
|
||||
using IRaCIS.Core.SCP.Service;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Localization;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Drawing;
|
||||
using System.Threading.Channels;
|
||||
using System.Threading.Tasks;
|
||||
using ZiggyCreatures.Caching.Fusion;
|
||||
|
||||
namespace IRaCIS.Core.SCP.Service;
|
||||
|
||||
public class FileUploadRecordAddOrEdit
|
||||
{
|
||||
public Guid? Id { get; set; }
|
||||
|
||||
|
||||
public string FileName { get; set; }
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public string FileType { get; set; }
|
||||
|
||||
public string Path { get; set; }
|
||||
|
||||
|
||||
public string UploadBatchId { get; set; }
|
||||
public BatchDataType BatchDataType { get; set; }
|
||||
|
||||
|
||||
public Guid? TrialId { get; set; }
|
||||
|
||||
public Guid? SubjectId { get; set; }
|
||||
|
||||
public Guid? SubjectVisitId { get; set; }
|
||||
|
||||
public Guid? DicomStudyId { get; set; }
|
||||
|
||||
public Guid? NoneDicomStudyId { get; set; }
|
||||
|
||||
|
||||
|
||||
public string FileMarkId { get; set; }
|
||||
|
||||
public int? Priority { get; set; }
|
||||
public string IP { get; set; }
|
||||
public bool? IsNeedSync { get; set; }
|
||||
public string UploadRegion { get; set; }
|
||||
public string TargetRegion { get; set; }
|
||||
}
|
||||
public interface IFileUploadRecordService
|
||||
{
|
||||
|
||||
|
||||
Task<IResponseOutput> AddOrUpdateFileUploadRecord(FileUploadRecordAddOrEdit addOrEditFileUploadRecord);
|
||||
|
||||
}
|
||||
|
||||
[ApiExplorerSettings(GroupName = "Common")]
|
||||
public class FileUploadRecordService(IRepository<FileUploadRecord> _fileUploadRecordRepository,
|
||||
IMapper _mapper, IUserInfo _userInfo, IStringLocalizer _localizer, IOptionsMonitor<ObjectStoreServiceOptions> options,
|
||||
IFusionCache _fusionCache, IRepository<Trial> _trialRepository, FileSyncQueue _fileSyncQueue) : BaseService, IFileUploadRecordService
|
||||
{
|
||||
|
||||
ObjectStoreServiceOptions ObjectStoreServiceConfig = options.CurrentValue;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public async Task<IResponseOutput> AddOrUpdateFileUploadRecord(FileUploadRecordAddOrEdit addOrEditFileUploadRecord)
|
||||
{
|
||||
|
||||
addOrEditFileUploadRecord.IP = _userInfo.IP;
|
||||
|
||||
if (ObjectStoreServiceConfig.IsOpenStoreSync && _userInfo.Domain.IsNotNullOrEmpty())
|
||||
{
|
||||
var find = ObjectStoreServiceConfig.SyncConfigList.FirstOrDefault(t => t.Domain == _userInfo.Domain);
|
||||
if (find != null)
|
||||
{
|
||||
addOrEditFileUploadRecord.UploadRegion = find.UploadRegion;
|
||||
addOrEditFileUploadRecord.TargetRegion = find.TargetRegion;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if (addOrEditFileUploadRecord.TrialId != null)
|
||||
{
|
||||
|
||||
var trialDataStore = await _fusionCache.GetOrSetAsync(CacheKeys.TrialDataStoreType(addOrEditFileUploadRecord.TrialId.Value), async _ =>
|
||||
{
|
||||
return await _trialRepository.Where(t => t.Id == addOrEditFileUploadRecord.TrialId).Select(t => t.TrialDataStoreType)
|
||||
.FirstOrDefaultAsync();
|
||||
},
|
||||
TimeSpan.FromDays(7)
|
||||
);
|
||||
|
||||
//项目配置了,那么就设置需要同步
|
||||
if (trialDataStore == TrialDataStore.MUtiCenter)
|
||||
{
|
||||
addOrEditFileUploadRecord.IsNeedSync = true;
|
||||
|
||||
addOrEditFileUploadRecord.Priority = 0;
|
||||
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
addOrEditFileUploadRecord.TargetRegion = "";
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
//系统文件,默认同步
|
||||
addOrEditFileUploadRecord.IsNeedSync = true;
|
||||
|
||||
addOrEditFileUploadRecord.Priority = 0;
|
||||
}
|
||||
|
||||
var entity = await _fileUploadRecordRepository.InsertOrUpdateAsync(addOrEditFileUploadRecord, true);
|
||||
|
||||
if (addOrEditFileUploadRecord.IsNeedSync == true)
|
||||
{
|
||||
_fileSyncQueue.Enqueue(entity.Id, addOrEditFileUploadRecord.Priority ?? 0);
|
||||
}
|
||||
|
||||
return ResponseOutput.Ok(entity.Id.ToString());
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 同步队列 信号量
|
||||
/// </summary>
|
||||
public class FileSyncQueue
|
||||
{
|
||||
private readonly PriorityQueue<Guid, int> _queue = new();
|
||||
private readonly SemaphoreSlim _signal = new(0);
|
||||
private readonly object _lock = new();
|
||||
|
||||
public void Enqueue(Guid id, int priority)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
// priority 越大越优先
|
||||
_queue.Enqueue(id, -priority);
|
||||
}
|
||||
|
||||
//类似于计数器,不会产生通知风暴,可消费资源 +1
|
||||
//if (有等待线程) 唤醒一个 else 仅增加计数
|
||||
_signal.Release(); // 唤醒一个 worker
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 如果没有任务 → 挂起等待 有任务 → 被唤醒并返回
|
||||
/// </summary>
|
||||
/// <param name="ct"></param>
|
||||
/// <returns></returns>
|
||||
public async Task<Guid> DequeueAsync(CancellationToken ct)
|
||||
{
|
||||
await _signal.WaitAsync(ct);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
return _queue.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
+1369
-105
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user