后端下载影像打成压缩包

IRaCIS.Core.Application/Service/Common/TrialImageDownloadService.cs

@@ -8,6 +8,7 @@ using FellowOakDicom.Imaging;
 using FellowOakDicom.Imaging.Render;
 using FellowOakDicom.IO.Buffer;
 using IRaCIS.Core.Application.Helper;
+using IRaCIS.Core.Application.MassTransit.Command;
 using IRaCIS.Core.Application.ViewModel;
 using IRaCIS.Core.Domain.Models;
 using IRaCIS.Core.Infrastructure;
@@ -18,17 +19,20 @@ using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc;
 using MiniExcelLibs;
 using NPOI.Util;
+using Org.BouncyCastle.Utilities.Zlib;
 using SharpCompress.Common;
 using System;
 using System.Collections.Generic;
 using System.Data;
 using System.Diagnostics;
+using System.IO.Compression;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using System.Web;
+using static IRaCIS.Core.Application.Service.TestService;
 using static IRaCIS.Core.Domain.Share.StaticData;
 using static Microsoft.EntityFrameworkCore.DbLoggerCategory;
 
@@ -97,6 +101,68 @@ namespace IRaCIS.Core.Application.Service
         }
 
 
+
+        public static class DownloadLogger
+        {
+
+
+            public static void Write(
+                string logFilePath,
+                string subjectCode,
+                decimal visitNum,
+                string visitName,
+                string? message = null)
+            {
+
+                bool fileExists = File.Exists(logFilePath);
+
+                using var stream = new FileStream(
+                    logFilePath,
+                    FileMode.Append,
+                    FileAccess.Write,
+                    FileShare.ReadWrite);
+
+                using var writer = new StreamWriter(stream, Encoding.UTF8);
+
+
+                // 首次写入表头
+                if (!fileExists)
+                {
+                    writer.WriteLine("Time,SubjectCode,VisitNum,VisitName,Message");
+                }
+
+                string line = string.Join(",",
+                [
+                Escape(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")),
+            subjectCode,
+            visitNum,
+            visitName,
+            Escape(message)
+                ]);
+
+                writer.WriteLine(line);
+            }
+
+            // 防止逗号、换行导致 CSV 错乱
+            private static string Escape(string? value)
+            {
+                if (string.IsNullOrEmpty(value))
+                    return "";
+
+                value = value.Replace("\"", "\"\"");
+
+                return $"\"{value}\"";
+            }
+        }
+
+        public class DownloadJob
+        {
+            public string Name { get; set; }
+
+
+            public Func<Task> Action { get; set; }
+        }
+
         /// <summary>
         /// 后端api swagger 下载项目影像
         /// </summary>
@@ -114,13 +180,14 @@ namespace IRaCIS.Core.Application.Service
             {
                 t.ResearchProgramNo,
 
-                VisitList = t.SubjectVisitList
+                VisitList = t.SubjectVisitList.Where(t => t.VisitTaskList.Any(t => t.TaskState == TaskState.Effect && t.ReadingCategory == ReadingCategory.Visit && t.ReadingTaskState != ReadingTaskState.HaveSigned && t.SourceSubjectVisitId != null && t.DoctorUserId != null))
                 //.Where(t=>subjectCodeList.Contains(t.Subject.Code))
                 .Select(sv => new
                 {
                     TrialSiteCode = sv.TrialSite.TrialSiteCode,
                     SubjectCode = sv.Subject.Code,
                     VisitName = sv.VisitName,
+                    VisitNum = sv.VisitNum,
                     StudyList = sv.StudyList.Select(u => new
                     {
                         u.PatientId,
@@ -154,159 +221,243 @@ namespace IRaCIS.Core.Application.Service
                             file.FileType
                         }).ToList()
                     }).ToList()
-                }).ToList()
+                }).OrderBy(t => t.SubjectCode).ThenBy(t => t.VisitNum).ToList()
 
             }).FirstOrDefault();
 
+            if (downloadInfo == null)
+            {
+                return ResponseOutput.Ok();
+            }
 
-            var count = downloadInfo.VisitList.SelectMany(t => t.NoneDicomStudyList).SelectMany(t => t.FileList).Count();
+            #region 设置目录
-            var count2 = downloadInfo.VisitList.SelectMany(t => t.StudyList).SelectMany(t => t.SeriesList).SelectMany(t => t.InstancePathList).Count();
+
+            var rootFolder = FileStoreHelper.GetIRaCISRootDataFolder(_hostEnvironment);
+            Directory.CreateDirectory(rootFolder);
+
+            // 获取无效字符（系统定义的）
+            string invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
+
+            // 用正则表达式替换所有非法字符为下划线或空字符
+            string pattern = $"[{Regex.Escape(invalidChars)}]";
+
+            var regexNo = Regex.Replace(downloadInfo.ResearchProgramNo, pattern, "_");
+
+            // 创建一个临时文件夹来存放文件
+            string trialFolderPath = Path.Combine(rootFolder, $"{regexNo}");//_{NewId.NextGuid()}
+            Directory.CreateDirectory(trialFolderPath);
+
+            var oldVisits = MiniExcel.Query<SubjectVisitExcel>(Path.Combine(rootFolder, "Old.xlsx")).ToList();
+
+            var acturalDownList = downloadInfo.VisitList.Where(t => !oldVisits.Any(old => old.VisitNum == t.VisitNum && old.SubjectCode == t.SubjectCode &&
+            old.VisitName.Trim().ToLower() == t.VisitName.Trim().ToLower())).ToList();
+
+            #endregion
+
+            #region 排除已经下载的
+
+            var logFilePath = Path.Combine(rootFolder, $"{trialId}_{regexNo}_download_log.csv");
+
+            if (File.Exists(logFilePath))
+            {
+                var existVisits = MiniExcel.Query<SubjectVisitExcel>(logFilePath, configuration: new MiniExcelLibs.Csv.CsvConfiguration()
+                {
+                    StreamReaderFunc = (stream) => new StreamReader(stream, Encoding.GetEncoding("gb2312"))
+                }).ToList();
+
+                acturalDownList = acturalDownList.Where(t => !existVisits.Any(old => old.VisitNum == t.VisitNum && old.SubjectCode == t.SubjectCode &&
+               old.VisitName.Trim().ToLower() == t.VisitName.Trim().ToLower())).ToList();
+
+
+            }
+
+            #endregion
+
+            var count = acturalDownList.SelectMany(t => t.NoneDicomStudyList).SelectMany(t => t.FileList).Count();
+
+            var count2 = acturalDownList.SelectMany(t => t.StudyList).SelectMany(t => t.SeriesList).SelectMany(t => t.InstancePathList).Count();
 
             Console.WriteLine($"下载总数量：{count}+{count2}={count + count2}");
 
-            if (downloadInfo != null)
+            var downloadJobs = new List<DownloadJob>();
+
+            foreach (var visitItem in acturalDownList)
             {
-                var downloadJobs = new List<Func<Task>>();
+                if (visitItem.StudyList.Count() == 0 && visitItem.NoneDicomStudyList.Count() == 0)
-
-                //var rootFolder = @"E:\DownloadImage";
-
-                var rootFolder = FileStoreHelper.GetDonwnloadImageFolder(_hostEnvironment);
-
-                // 获取无效字符（系统定义的）
-                string invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
-
-                // 用正则表达式替换所有非法字符为下划线或空字符
-                string pattern = $"[{Regex.Escape(invalidChars)}]";
-
-                var regexNo = Regex.Replace(downloadInfo.ResearchProgramNo, pattern, "_");
-
-                // 创建一个临时文件夹来存放文件
-                string trialFolderPath = Path.Combine(rootFolder, $"{regexNo}_{NewId.NextGuid()}");
-                Directory.CreateDirectory(trialFolderPath);
-
-                foreach (var visitItem in downloadInfo.VisitList)
                 {
-                    if (visitItem.StudyList.Count() == 0 && visitItem.NoneDicomStudyList.Count() == 0)
+                    continue;
+                }
+
+                #region 处理 中心，受试者dicom non-dicom 文件夹层级
+
+                //var siteFolderPath = Path.Combine(trialFolderPath, visitItem.TrialSiteCode);
+                //if (!Directory.Exists(siteFolderPath))
+                //{
+                //    Directory.CreateDirectory(siteFolderPath);
+                //}
+
+                #endregion
+
+
+                foreach (var studyInfo in visitItem.StudyList)
+                {
+                    // 遍历 Series
+                    foreach (var seriesInfo in studyInfo.SeriesList)
                     {
-                        continue;
+                        string studyDicomFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}", $"{studyInfo.StudyCode}_{studyInfo.StudyTime?.ToString("yyyy-MM-dd")}_{seriesInfo.Modality}");
-                    }
 
-                    #region 处理 中心，受试者dicom non-dicom 文件夹层级
+                        // 创建 影像 文件夹
+                        Directory.CreateDirectory(studyDicomFolderPath);
 
-                    var siteFolderPath = Path.Combine(trialFolderPath, visitItem.TrialSiteCode);
+                        // 遍历 InstancePathList
-                    if (!Directory.Exists(siteFolderPath))
+                        foreach (var instanceInfo in seriesInfo.InstancePathList)
-                    {
-                        Directory.CreateDirectory(siteFolderPath);
-                    }
-
-                    #endregion
-
-
-                    foreach (var studyInfo in visitItem.StudyList)
-                    {
-                        // 遍历 Series
-                        foreach (var seriesInfo in studyInfo.SeriesList)
                         {
-                            string studyDicomFolderPath = Path.Combine(siteFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}_DICOM", $"{studyInfo.StudyCode}_{studyInfo.StudyTime?.ToString("yyyy-MM-dd")}_{seriesInfo.Modality}");
+                            // 复制文件到相应的文件夹
+                            string destinationPath = Path.Combine(studyDicomFolderPath, Path.GetFileName(instanceInfo.Path));
 
-                            // 创建 影像 文件夹
-                            Directory.CreateDirectory(studyDicomFolderPath);
 
-                            // 遍历 InstancePathList
+                            downloadJobs.Add(new DownloadJob()
-                            foreach (var instanceInfo in seriesInfo.InstancePathList)
                             {
-                                // 复制文件到相应的文件夹
+                                Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_DICOM_{destinationPath}",
-                                string destinationPath = Path.Combine(studyDicomFolderPath, Path.GetFileName(instanceInfo.Path));
 
+                                Action = async () =>
+                            {
                                 await using var output = File.Create(destinationPath);
+
                                 if (instanceInfo.IsEncapsulated)
                                 {
-                                    //加入到下载任务里
+                                    await TryWriteMergedDicomAsync(
-                                    downloadJobs.Add(() => TryWriteMergedDicomAsync(() => _oSSService.GetStreamFromOSSAsync(instanceInfo.Path), output));
+                                        () => _oSSService.GetStreamFromOSSAsync(instanceInfo.Path),
+                                        output);
                                 }
                                 else
                                 {
-                                    //加入到下载任务里
+                                    await using var input =
-                                    downloadJobs.Add(() => _oSSService.DownLoadFromOSSAsync(instanceInfo.Path, destinationPath));
+                                        await _oSSService.GetStreamFromOSSAsync(instanceInfo.Path);
+
+                                    await input.CopyToAsync(output);
                                 }
-                               
-
-                                //下载到当前目录
-                                //await _oSSService.DownLoadFromOSSAsync(instanceInfo.Path, destinationPath);
                             }
-                        }
+                            });
 
-
-                    }
-
-                    foreach (var noneDicomStudy in visitItem.NoneDicomStudyList)
-                    {
-                        string studyNoneDicomFolderPath = Path.Combine(siteFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}_Non-DICOM", $"{noneDicomStudy.StudyCode}_{noneDicomStudy.ImageDate.ToString("yyyy-MM-dd")}_{noneDicomStudy.Modality}");
-
-                        Directory.CreateDirectory(studyNoneDicomFolderPath);
-
-                        foreach (var file in noneDicomStudy.FileList)
-                        {
-                            string destinationPath = Path.Combine(studyNoneDicomFolderPath, Path.GetFileName(file.FileName));
-
-                            //加入到下载任务里
-                            downloadJobs.Add(() => _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath));
-                            //下载到当前目录
-                            //await _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath);
                         }
                     }
 
 
                 }
 
-                #region 异步方式处理
+                foreach (var noneDicomStudy in visitItem.NoneDicomStudyList)
-
-                int totalCount = downloadJobs.Count;
-                int downloadedCount = 0;
-
-                foreach (var job in downloadJobs)
                 {
-                    try
+                    string studyNoneDicomFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}", $"{noneDicomStudy.StudyCode}_{noneDicomStudy.ImageDate.ToString("yyyy-MM-dd")}_{noneDicomStudy.Modality}");
+
+                    Directory.CreateDirectory(studyNoneDicomFolderPath);
+
+                    foreach (var file in noneDicomStudy.FileList)
                     {
-                        await job();
+                        string destinationPath = Path.Combine(studyNoneDicomFolderPath, Path.GetFileName(file.FileName));
-                    }
-                    catch (Exception ex)
-                    {
-                        Log.Logger.Error($"下载失败: {ex.Message}");
-                    }
 
-                    downloadedCount++;
+                        //加入到下载任务里
-
+                        downloadJobs.Add(new DownloadJob() { Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_NoneDICOM_{destinationPath}", Action = () => _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath) });
-                    // 每处理50个，输出一次进度（或最后一个时也输出）
-                    if (downloadedCount % 50 == 0 || downloadedCount == totalCount)
-                    {
-
-                        Log.Logger.Error($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
 
+                        //下载到当前目录
+                        //await _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath);
                     }
                 }
-                #endregion
 
-                #region 多线程测试
 
-                //const int batchSize = 15;
+                //建立压缩包
-                //int totalCount = downloadJobs.Count;
+                string visitFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}");
-                //int downloadedCount = 0;
 
-                //for (int i = 0; i < downloadJobs.Count; i += batchSize)
+                downloadJobs.Add(new DownloadJob()
-                //{
+                {
-                //    var batch = downloadJobs.Skip(i).Take(batchSize).Select(job => job());
+                    Name = $"{visitItem.SubjectCode}_{visitItem.VisitName}_Zip",
 
-                //    await Task.WhenAll(batch);
+                    Action = async () =>
+                    {
+                        string zipPath = visitFolderPath + ".zip";
 
-                //    downloadedCount += batch.Count();
+                        if (File.Exists(zipPath))
+                        {
+                            File.Delete(zipPath);
+                        }
 
-                //    Console.WriteLine($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
+                        ZipFile.CreateFromDirectory(
-                //}
+                            visitFolderPath,
-                #endregion
+                            zipPath,
+                            CompressionLevel.Fastest,
+                            false);
 
+                        Directory.Delete(visitFolderPath, true);
+
+                        await Task.CompletedTask;
+                    }
+                });
+
+                //记录日志
+                downloadJobs.Add(new DownloadJob()
+                {
+                    Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_Finished",
+                    Action = () =>
+                {
+                    DownloadLogger.Write(
+                        logFilePath: logFilePath,
+                        subjectCode: visitItem.SubjectCode,
+                        visitNum: visitItem.VisitNum,
+                        visitName: visitItem.VisitName,
+                        message: "Success");
+
+                    return Task.CompletedTask;
+                }
+                });
             }
 
+            #region 异步方式处理
+
+            int totalCount = downloadJobs.Count;
+            int downloadedCount = 0;
+
+            foreach (var job in downloadJobs)
+            {
+                try
+                {
+                    await job.Action();
+                }
+                catch (Exception ex)
+                {
+                    Log.Logger.Error($"{job.Name}下载失败: {ex.Message}");
+                }
+
+                downloadedCount++;
+
+                // 每处理50个，输出一次进度（或最后一个时也输出）
+                if (downloadedCount % 50 == 0 || downloadedCount == totalCount)
+                {
+
+                    Log.Logger.Warning($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
+
+                }
+            }
+            #endregion
+
+            #region 多线程测试
+
+            //const int batchSize = 15;
+            //int totalCount = downloadJobs.Count;
+            //int downloadedCount = 0;
+
+            //for (int i = 0; i < downloadJobs.Count; i += batchSize)
+            //{
+            //    var batch = downloadJobs.Skip(i).Take(batchSize).Select(job => job());
+
+            //    await Task.WhenAll(batch);
+
+            //    downloadedCount += batch.Count();
+
+            //    Console.WriteLine($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
+            //}
+            #endregion
+
+
             return ResponseOutput.Ok();