diff --git a/IRaCIS.Core.Application/Service/Common/TrialImageDownloadService.cs b/IRaCIS.Core.Application/Service/Common/TrialImageDownloadService.cs index a23703c16..faf9d5274 100644 --- a/IRaCIS.Core.Application/Service/Common/TrialImageDownloadService.cs +++ b/IRaCIS.Core.Application/Service/Common/TrialImageDownloadService.cs @@ -8,6 +8,7 @@ using FellowOakDicom.Imaging; using FellowOakDicom.Imaging.Render; using FellowOakDicom.IO.Buffer; using IRaCIS.Core.Application.Helper; +using IRaCIS.Core.Application.MassTransit.Command; using IRaCIS.Core.Application.ViewModel; using IRaCIS.Core.Domain.Models; using IRaCIS.Core.Infrastructure; @@ -18,17 +19,20 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; using MiniExcelLibs; using NPOI.Util; +using Org.BouncyCastle.Utilities.Zlib; using SharpCompress.Common; using System; using System.Collections.Generic; using System.Data; using System.Diagnostics; +using System.IO.Compression; using System.Linq; using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; +using static IRaCIS.Core.Application.Service.TestService; using static IRaCIS.Core.Domain.Share.StaticData; using static Microsoft.EntityFrameworkCore.DbLoggerCategory; @@ -97,6 +101,68 @@ namespace IRaCIS.Core.Application.Service } + + public static class DownloadLogger + { + + + public static void Write( + string logFilePath, + string subjectCode, + decimal visitNum, + string visitName, + string? message = null) + { + + bool fileExists = File.Exists(logFilePath); + + using var stream = new FileStream( + logFilePath, + FileMode.Append, + FileAccess.Write, + FileShare.ReadWrite); + + using var writer = new StreamWriter(stream, Encoding.UTF8); + + + // 首次写入表头 + if (!fileExists) + { + writer.WriteLine("Time,SubjectCode,VisitNum,VisitName,Message"); + } + + string line = string.Join(",", + [ + Escape(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")), + subjectCode, + visitNum, + visitName, + Escape(message) + ]); + + writer.WriteLine(line); + } + + // 防止逗号、换行导致 CSV 错乱 + private static string Escape(string? value) + { + if (string.IsNullOrEmpty(value)) + return ""; + + value = value.Replace("\"", "\"\""); + + return $"\"{value}\""; + } + } + + public class DownloadJob + { + public string Name { get; set; } + + + public Func Action { get; set; } + } + /// /// 后端api swagger 下载项目影像 /// @@ -114,13 +180,14 @@ namespace IRaCIS.Core.Application.Service { t.ResearchProgramNo, - VisitList = t.SubjectVisitList + VisitList = t.SubjectVisitList.Where(t => t.VisitTaskList.Any(t => t.TaskState == TaskState.Effect && t.ReadingCategory == ReadingCategory.Visit && t.ReadingTaskState != ReadingTaskState.HaveSigned && t.SourceSubjectVisitId != null && t.DoctorUserId != null)) //.Where(t=>subjectCodeList.Contains(t.Subject.Code)) .Select(sv => new { TrialSiteCode = sv.TrialSite.TrialSiteCode, SubjectCode = sv.Subject.Code, VisitName = sv.VisitName, + VisitNum = sv.VisitNum, StudyList = sv.StudyList.Select(u => new { u.PatientId, @@ -154,159 +221,243 @@ namespace IRaCIS.Core.Application.Service file.FileType }).ToList() }).ToList() - }).ToList() + }).OrderBy(t => t.SubjectCode).ThenBy(t => t.VisitNum).ToList() }).FirstOrDefault(); + if (downloadInfo == null) + { + return ResponseOutput.Ok(); + } - var count = downloadInfo.VisitList.SelectMany(t => t.NoneDicomStudyList).SelectMany(t => t.FileList).Count(); - var count2 = downloadInfo.VisitList.SelectMany(t => t.StudyList).SelectMany(t => t.SeriesList).SelectMany(t => t.InstancePathList).Count(); + #region 设置目录 + + var rootFolder = FileStoreHelper.GetIRaCISRootDataFolder(_hostEnvironment); + Directory.CreateDirectory(rootFolder); + + // 获取无效字符（系统定义的） + string invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()); + + // 用正则表达式替换所有非法字符为下划线或空字符 + string pattern = $"[{Regex.Escape(invalidChars)}]"; + + var regexNo = Regex.Replace(downloadInfo.ResearchProgramNo, pattern, "_"); + + // 创建一个临时文件夹来存放文件 + string trialFolderPath = Path.Combine(rootFolder, $"{regexNo}");//_{NewId.NextGuid()} + Directory.CreateDirectory(trialFolderPath); + + var oldVisits = MiniExcel.Query(Path.Combine(rootFolder, "Old.xlsx")).ToList(); + + var acturalDownList = downloadInfo.VisitList.Where(t => !oldVisits.Any(old => old.VisitNum == t.VisitNum && old.SubjectCode == t.SubjectCode && + old.VisitName.Trim().ToLower() == t.VisitName.Trim().ToLower())).ToList(); + + #endregion + + #region 排除已经下载的 + + var logFilePath = Path.Combine(rootFolder, $"{trialId}_{regexNo}_download_log.csv"); + + if (File.Exists(logFilePath)) + { + var existVisits = MiniExcel.Query(logFilePath, configuration: new MiniExcelLibs.Csv.CsvConfiguration() + { + StreamReaderFunc = (stream) => new StreamReader(stream, Encoding.GetEncoding("gb2312")) + }).ToList(); + + acturalDownList = acturalDownList.Where(t => !existVisits.Any(old => old.VisitNum == t.VisitNum && old.SubjectCode == t.SubjectCode && + old.VisitName.Trim().ToLower() == t.VisitName.Trim().ToLower())).ToList(); + + + } + + #endregion + + var count = acturalDownList.SelectMany(t => t.NoneDicomStudyList).SelectMany(t => t.FileList).Count(); + + var count2 = acturalDownList.SelectMany(t => t.StudyList).SelectMany(t => t.SeriesList).SelectMany(t => t.InstancePathList).Count(); Console.WriteLine($"下载总数量：{count}+{count2}={count + count2}"); - if (downloadInfo != null) + var downloadJobs = new List(); + + foreach (var visitItem in acturalDownList) { - var downloadJobs = new List>(); - - //var rootFolder = @"E:\DownloadImage"; - - var rootFolder = FileStoreHelper.GetDonwnloadImageFolder(_hostEnvironment); - - // 获取无效字符（系统定义的） - string invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()); - - // 用正则表达式替换所有非法字符为下划线或空字符 - string pattern = $"[{Regex.Escape(invalidChars)}]"; - - var regexNo = Regex.Replace(downloadInfo.ResearchProgramNo, pattern, "_"); - - // 创建一个临时文件夹来存放文件 - string trialFolderPath = Path.Combine(rootFolder, $"{regexNo}_{NewId.NextGuid()}"); - Directory.CreateDirectory(trialFolderPath); - - foreach (var visitItem in downloadInfo.VisitList) + if (visitItem.StudyList.Count() == 0 && visitItem.NoneDicomStudyList.Count() == 0) { - if (visitItem.StudyList.Count() == 0 && visitItem.NoneDicomStudyList.Count() == 0) + continue; + } + + #region 处理 中心，受试者dicom non-dicom 文件夹层级 + + //var siteFolderPath = Path.Combine(trialFolderPath, visitItem.TrialSiteCode); + //if (!Directory.Exists(siteFolderPath)) + //{ + // Directory.CreateDirectory(siteFolderPath); + //} + + #endregion + + + foreach (var studyInfo in visitItem.StudyList) + { + // 遍历 Series + foreach (var seriesInfo in studyInfo.SeriesList) { - continue; - } + string studyDicomFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}", $"{studyInfo.StudyCode}_{studyInfo.StudyTime?.ToString("yyyy-MM-dd")}_{seriesInfo.Modality}"); - #region 处理 中心，受试者dicom non-dicom 文件夹层级 + // 创建 影像 文件夹 + Directory.CreateDirectory(studyDicomFolderPath); - var siteFolderPath = Path.Combine(trialFolderPath, visitItem.TrialSiteCode); - if (!Directory.Exists(siteFolderPath)) - { - Directory.CreateDirectory(siteFolderPath); - } - - #endregion - - - foreach (var studyInfo in visitItem.StudyList) - { - // 遍历 Series - foreach (var seriesInfo in studyInfo.SeriesList) + // 遍历 InstancePathList + foreach (var instanceInfo in seriesInfo.InstancePathList) { - string studyDicomFolderPath = Path.Combine(siteFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}_DICOM", $"{studyInfo.StudyCode}_{studyInfo.StudyTime?.ToString("yyyy-MM-dd")}_{seriesInfo.Modality}"); + // 复制文件到相应的文件夹 + string destinationPath = Path.Combine(studyDicomFolderPath, Path.GetFileName(instanceInfo.Path)); - // 创建 影像 文件夹 - Directory.CreateDirectory(studyDicomFolderPath); - // 遍历 InstancePathList - foreach (var instanceInfo in seriesInfo.InstancePathList) + downloadJobs.Add(new DownloadJob() { - // 复制文件到相应的文件夹 - string destinationPath = Path.Combine(studyDicomFolderPath, Path.GetFileName(instanceInfo.Path)); + Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_DICOM_{destinationPath}", + Action = async () => + { await using var output = File.Create(destinationPath); + if (instanceInfo.IsEncapsulated) { - //加入到下载任务里 - downloadJobs.Add(() => TryWriteMergedDicomAsync(() => _oSSService.GetStreamFromOSSAsync(instanceInfo.Path), output)); + await TryWriteMergedDicomAsync( + () => _oSSService.GetStreamFromOSSAsync(instanceInfo.Path), + output); } else { - //加入到下载任务里 - downloadJobs.Add(() => _oSSService.DownLoadFromOSSAsync(instanceInfo.Path, destinationPath)); + await using var input = + await _oSSService.GetStreamFromOSSAsync(instanceInfo.Path); + + await input.CopyToAsync(output); } - - - //下载到当前目录 - //await _oSSService.DownLoadFromOSSAsync(instanceInfo.Path, destinationPath); } - } + }); - - } - - foreach (var noneDicomStudy in visitItem.NoneDicomStudyList) - { - string studyNoneDicomFolderPath = Path.Combine(siteFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}_Non-DICOM", $"{noneDicomStudy.StudyCode}_{noneDicomStudy.ImageDate.ToString("yyyy-MM-dd")}_{noneDicomStudy.Modality}"); - - Directory.CreateDirectory(studyNoneDicomFolderPath); - - foreach (var file in noneDicomStudy.FileList) - { - string destinationPath = Path.Combine(studyNoneDicomFolderPath, Path.GetFileName(file.FileName)); - - //加入到下载任务里 - downloadJobs.Add(() => _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath)); - //下载到当前目录 - //await _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath); } } } - #region 异步方式处理 - - int totalCount = downloadJobs.Count; - int downloadedCount = 0; - - foreach (var job in downloadJobs) + foreach (var noneDicomStudy in visitItem.NoneDicomStudyList) { - try + string studyNoneDicomFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}", $"{noneDicomStudy.StudyCode}_{noneDicomStudy.ImageDate.ToString("yyyy-MM-dd")}_{noneDicomStudy.Modality}"); + + Directory.CreateDirectory(studyNoneDicomFolderPath); + + foreach (var file in noneDicomStudy.FileList) { - await job(); - } - catch (Exception ex) - { - Log.Logger.Error($"下载失败: {ex.Message}"); - } + string destinationPath = Path.Combine(studyNoneDicomFolderPath, Path.GetFileName(file.FileName)); - downloadedCount++; - - // 每处理50个，输出一次进度（或最后一个时也输出） - if (downloadedCount % 50 == 0 || downloadedCount == totalCount) - { - - Log.Logger.Error($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%"); + //加入到下载任务里 + downloadJobs.Add(new DownloadJob() { Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_NoneDICOM_{destinationPath}", Action = () => _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath) }); + //下载到当前目录 + //await _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath); } } - #endregion - #region 多线程测试 - //const int batchSize = 15; - //int totalCount = downloadJobs.Count; - //int downloadedCount = 0; + //建立压缩包 + string visitFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}"); - //for (int i = 0; i < downloadJobs.Count; i += batchSize) - //{ - // var batch = downloadJobs.Skip(i).Take(batchSize).Select(job => job()); + downloadJobs.Add(new DownloadJob() + { + Name = $"{visitItem.SubjectCode}_{visitItem.VisitName}_Zip", - // await Task.WhenAll(batch); + Action = async () => + { + string zipPath = visitFolderPath + ".zip"; - // downloadedCount += batch.Count(); + if (File.Exists(zipPath)) + { + File.Delete(zipPath); + } - // Console.WriteLine($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%"); - //} - #endregion + ZipFile.CreateFromDirectory( + visitFolderPath, + zipPath, + CompressionLevel.Fastest, + false); + Directory.Delete(visitFolderPath, true); + + await Task.CompletedTask; + } + }); + + //记录日志 + downloadJobs.Add(new DownloadJob() + { + Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_Finished", + Action = () => + { + DownloadLogger.Write( + logFilePath: logFilePath, + subjectCode: visitItem.SubjectCode, + visitNum: visitItem.VisitNum, + visitName: visitItem.VisitName, + message: "Success"); + + return Task.CompletedTask; + } + }); } + #region 异步方式处理 + + int totalCount = downloadJobs.Count; + int downloadedCount = 0; + + foreach (var job in downloadJobs) + { + try + { + await job.Action(); + } + catch (Exception ex) + { + Log.Logger.Error($"{job.Name}下载失败: {ex.Message}"); + } + + downloadedCount++; + + // 每处理50个，输出一次进度（或最后一个时也输出） + if (downloadedCount % 50 == 0 || downloadedCount == totalCount) + { + + Log.Logger.Warning($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%"); + + } + } + #endregion + + #region 多线程测试 + + //const int batchSize = 15; + //int totalCount = downloadJobs.Count; + //int downloadedCount = 0; + + //for (int i = 0; i < downloadJobs.Count; i += batchSize) + //{ + // var batch = downloadJobs.Skip(i).Take(batchSize).Select(job => job()); + + // await Task.WhenAll(batch); + + // downloadedCount += batch.Count(); + + // Console.WriteLine($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%"); + //} + #endregion + + return ResponseOutput.Ok();