后端下载影像打成压缩包
parent
d7da7b69d4
commit
40f744ea4a
|
|
@ -8,6 +8,7 @@ using FellowOakDicom.Imaging;
|
|||
using FellowOakDicom.Imaging.Render;
|
||||
using FellowOakDicom.IO.Buffer;
|
||||
using IRaCIS.Core.Application.Helper;
|
||||
using IRaCIS.Core.Application.MassTransit.Command;
|
||||
using IRaCIS.Core.Application.ViewModel;
|
||||
using IRaCIS.Core.Domain.Models;
|
||||
using IRaCIS.Core.Infrastructure;
|
||||
|
|
@ -18,17 +19,20 @@ using Microsoft.AspNetCore.Hosting;
|
|||
using Microsoft.AspNetCore.Mvc;
|
||||
using MiniExcelLibs;
|
||||
using NPOI.Util;
|
||||
using Org.BouncyCastle.Utilities.Zlib;
|
||||
using SharpCompress.Common;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Diagnostics;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
using static IRaCIS.Core.Application.Service.TestService;
|
||||
using static IRaCIS.Core.Domain.Share.StaticData;
|
||||
using static Microsoft.EntityFrameworkCore.DbLoggerCategory;
|
||||
|
||||
|
|
@ -97,6 +101,68 @@ namespace IRaCIS.Core.Application.Service
|
|||
}
|
||||
|
||||
|
||||
|
||||
public static class DownloadLogger
|
||||
{
|
||||
|
||||
|
||||
public static void Write(
|
||||
string logFilePath,
|
||||
string subjectCode,
|
||||
decimal visitNum,
|
||||
string visitName,
|
||||
string? message = null)
|
||||
{
|
||||
|
||||
bool fileExists = File.Exists(logFilePath);
|
||||
|
||||
using var stream = new FileStream(
|
||||
logFilePath,
|
||||
FileMode.Append,
|
||||
FileAccess.Write,
|
||||
FileShare.ReadWrite);
|
||||
|
||||
using var writer = new StreamWriter(stream, Encoding.UTF8);
|
||||
|
||||
|
||||
// 首次写入表头
|
||||
if (!fileExists)
|
||||
{
|
||||
writer.WriteLine("Time,SubjectCode,VisitNum,VisitName,Message");
|
||||
}
|
||||
|
||||
string line = string.Join(",",
|
||||
[
|
||||
Escape(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")),
|
||||
subjectCode,
|
||||
visitNum,
|
||||
visitName,
|
||||
Escape(message)
|
||||
]);
|
||||
|
||||
writer.WriteLine(line);
|
||||
}
|
||||
|
||||
// 防止逗号、换行导致 CSV 错乱
|
||||
private static string Escape(string? value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value))
|
||||
return "";
|
||||
|
||||
value = value.Replace("\"", "\"\"");
|
||||
|
||||
return $"\"{value}\"";
|
||||
}
|
||||
}
|
||||
|
||||
public class DownloadJob
|
||||
{
|
||||
public string Name { get; set; }
|
||||
|
||||
|
||||
public Func<Task> Action { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 后端api swagger 下载项目影像
|
||||
/// </summary>
|
||||
|
|
@ -114,13 +180,14 @@ namespace IRaCIS.Core.Application.Service
|
|||
{
|
||||
t.ResearchProgramNo,
|
||||
|
||||
VisitList = t.SubjectVisitList
|
||||
VisitList = t.SubjectVisitList.Where(t => t.VisitTaskList.Any(t => t.TaskState == TaskState.Effect && t.ReadingCategory == ReadingCategory.Visit && t.ReadingTaskState != ReadingTaskState.HaveSigned && t.SourceSubjectVisitId != null && t.DoctorUserId != null))
|
||||
//.Where(t=>subjectCodeList.Contains(t.Subject.Code))
|
||||
.Select(sv => new
|
||||
{
|
||||
TrialSiteCode = sv.TrialSite.TrialSiteCode,
|
||||
SubjectCode = sv.Subject.Code,
|
||||
VisitName = sv.VisitName,
|
||||
VisitNum = sv.VisitNum,
|
||||
StudyList = sv.StudyList.Select(u => new
|
||||
{
|
||||
u.PatientId,
|
||||
|
|
@ -154,159 +221,243 @@ namespace IRaCIS.Core.Application.Service
|
|||
file.FileType
|
||||
}).ToList()
|
||||
}).ToList()
|
||||
}).ToList()
|
||||
}).OrderBy(t => t.SubjectCode).ThenBy(t => t.VisitNum).ToList()
|
||||
|
||||
}).FirstOrDefault();
|
||||
|
||||
if (downloadInfo == null)
|
||||
{
|
||||
return ResponseOutput.Ok();
|
||||
}
|
||||
|
||||
var count = downloadInfo.VisitList.SelectMany(t => t.NoneDicomStudyList).SelectMany(t => t.FileList).Count();
|
||||
var count2 = downloadInfo.VisitList.SelectMany(t => t.StudyList).SelectMany(t => t.SeriesList).SelectMany(t => t.InstancePathList).Count();
|
||||
#region 设置目录
|
||||
|
||||
var rootFolder = FileStoreHelper.GetIRaCISRootDataFolder(_hostEnvironment);
|
||||
Directory.CreateDirectory(rootFolder);
|
||||
|
||||
// 获取无效字符（系统定义的）
|
||||
string invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
|
||||
|
||||
// 用正则表达式替换所有非法字符为下划线或空字符
|
||||
string pattern = $"[{Regex.Escape(invalidChars)}]";
|
||||
|
||||
var regexNo = Regex.Replace(downloadInfo.ResearchProgramNo, pattern, "_");
|
||||
|
||||
// 创建一个临时文件夹来存放文件
|
||||
string trialFolderPath = Path.Combine(rootFolder, $"{regexNo}");//_{NewId.NextGuid()}
|
||||
Directory.CreateDirectory(trialFolderPath);
|
||||
|
||||
var oldVisits = MiniExcel.Query<SubjectVisitExcel>(Path.Combine(rootFolder, "Old.xlsx")).ToList();
|
||||
|
||||
var acturalDownList = downloadInfo.VisitList.Where(t => !oldVisits.Any(old => old.VisitNum == t.VisitNum && old.SubjectCode == t.SubjectCode &&
|
||||
old.VisitName.Trim().ToLower() == t.VisitName.Trim().ToLower())).ToList();
|
||||
|
||||
#endregion
|
||||
|
||||
#region 排除已经下载的
|
||||
|
||||
var logFilePath = Path.Combine(rootFolder, $"{trialId}_{regexNo}_download_log.csv");
|
||||
|
||||
if (File.Exists(logFilePath))
|
||||
{
|
||||
var existVisits = MiniExcel.Query<SubjectVisitExcel>(logFilePath, configuration: new MiniExcelLibs.Csv.CsvConfiguration()
|
||||
{
|
||||
StreamReaderFunc = (stream) => new StreamReader(stream, Encoding.GetEncoding("gb2312"))
|
||||
}).ToList();
|
||||
|
||||
acturalDownList = acturalDownList.Where(t => !existVisits.Any(old => old.VisitNum == t.VisitNum && old.SubjectCode == t.SubjectCode &&
|
||||
old.VisitName.Trim().ToLower() == t.VisitName.Trim().ToLower())).ToList();
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
var count = acturalDownList.SelectMany(t => t.NoneDicomStudyList).SelectMany(t => t.FileList).Count();
|
||||
|
||||
var count2 = acturalDownList.SelectMany(t => t.StudyList).SelectMany(t => t.SeriesList).SelectMany(t => t.InstancePathList).Count();
|
||||
|
||||
Console.WriteLine($"下载总数量：{count}+{count2}={count + count2}");
|
||||
|
||||
if (downloadInfo != null)
|
||||
var downloadJobs = new List<DownloadJob>();
|
||||
|
||||
foreach (var visitItem in acturalDownList)
|
||||
{
|
||||
var downloadJobs = new List<Func<Task>>();
|
||||
|
||||
//var rootFolder = @"E:\DownloadImage";
|
||||
|
||||
var rootFolder = FileStoreHelper.GetDonwnloadImageFolder(_hostEnvironment);
|
||||
|
||||
// 获取无效字符（系统定义的）
|
||||
string invalidChars = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
|
||||
|
||||
// 用正则表达式替换所有非法字符为下划线或空字符
|
||||
string pattern = $"[{Regex.Escape(invalidChars)}]";
|
||||
|
||||
var regexNo = Regex.Replace(downloadInfo.ResearchProgramNo, pattern, "_");
|
||||
|
||||
// 创建一个临时文件夹来存放文件
|
||||
string trialFolderPath = Path.Combine(rootFolder, $"{regexNo}_{NewId.NextGuid()}");
|
||||
Directory.CreateDirectory(trialFolderPath);
|
||||
|
||||
foreach (var visitItem in downloadInfo.VisitList)
|
||||
if (visitItem.StudyList.Count() == 0 && visitItem.NoneDicomStudyList.Count() == 0)
|
||||
{
|
||||
if (visitItem.StudyList.Count() == 0 && visitItem.NoneDicomStudyList.Count() == 0)
|
||||
continue;
|
||||
}
|
||||
|
||||
#region 处理 中心，受试者dicom non-dicom 文件夹层级
|
||||
|
||||
//var siteFolderPath = Path.Combine(trialFolderPath, visitItem.TrialSiteCode);
|
||||
//if (!Directory.Exists(siteFolderPath))
|
||||
//{
|
||||
// Directory.CreateDirectory(siteFolderPath);
|
||||
//}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
foreach (var studyInfo in visitItem.StudyList)
|
||||
{
|
||||
// 遍历 Series
|
||||
foreach (var seriesInfo in studyInfo.SeriesList)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string studyDicomFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}", $"{studyInfo.StudyCode}_{studyInfo.StudyTime?.ToString("yyyy-MM-dd")}_{seriesInfo.Modality}");
|
||||
|
||||
#region 处理 中心，受试者dicom non-dicom 文件夹层级
|
||||
// 创建 影像 文件夹
|
||||
Directory.CreateDirectory(studyDicomFolderPath);
|
||||
|
||||
var siteFolderPath = Path.Combine(trialFolderPath, visitItem.TrialSiteCode);
|
||||
if (!Directory.Exists(siteFolderPath))
|
||||
{
|
||||
Directory.CreateDirectory(siteFolderPath);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
foreach (var studyInfo in visitItem.StudyList)
|
||||
{
|
||||
// 遍历 Series
|
||||
foreach (var seriesInfo in studyInfo.SeriesList)
|
||||
// 遍历 InstancePathList
|
||||
foreach (var instanceInfo in seriesInfo.InstancePathList)
|
||||
{
|
||||
string studyDicomFolderPath = Path.Combine(siteFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}_DICOM", $"{studyInfo.StudyCode}_{studyInfo.StudyTime?.ToString("yyyy-MM-dd")}_{seriesInfo.Modality}");
|
||||
// 复制文件到相应的文件夹
|
||||
string destinationPath = Path.Combine(studyDicomFolderPath, Path.GetFileName(instanceInfo.Path));
|
||||
|
||||
// 创建 影像 文件夹
|
||||
Directory.CreateDirectory(studyDicomFolderPath);
|
||||
|
||||
// 遍历 InstancePathList
|
||||
foreach (var instanceInfo in seriesInfo.InstancePathList)
|
||||
downloadJobs.Add(new DownloadJob()
|
||||
{
|
||||
// 复制文件到相应的文件夹
|
||||
string destinationPath = Path.Combine(studyDicomFolderPath, Path.GetFileName(instanceInfo.Path));
|
||||
Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_DICOM_{destinationPath}",
|
||||
|
||||
Action = async () =>
|
||||
{
|
||||
await using var output = File.Create(destinationPath);
|
||||
|
||||
if (instanceInfo.IsEncapsulated)
|
||||
{
|
||||
//加入到下载任务里
|
||||
downloadJobs.Add(() => TryWriteMergedDicomAsync(() => _oSSService.GetStreamFromOSSAsync(instanceInfo.Path), output));
|
||||
await TryWriteMergedDicomAsync(
|
||||
() => _oSSService.GetStreamFromOSSAsync(instanceInfo.Path),
|
||||
output);
|
||||
}
|
||||
else
|
||||
{
|
||||
//加入到下载任务里
|
||||
downloadJobs.Add(() => _oSSService.DownLoadFromOSSAsync(instanceInfo.Path, destinationPath));
|
||||
await using var input =
|
||||
await _oSSService.GetStreamFromOSSAsync(instanceInfo.Path);
|
||||
|
||||
await input.CopyToAsync(output);
|
||||
}
|
||||
|
||||
|
||||
//下载到当前目录
|
||||
//await _oSSService.DownLoadFromOSSAsync(instanceInfo.Path, destinationPath);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
foreach (var noneDicomStudy in visitItem.NoneDicomStudyList)
|
||||
{
|
||||
string studyNoneDicomFolderPath = Path.Combine(siteFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}_Non-DICOM", $"{noneDicomStudy.StudyCode}_{noneDicomStudy.ImageDate.ToString("yyyy-MM-dd")}_{noneDicomStudy.Modality}");
|
||||
|
||||
Directory.CreateDirectory(studyNoneDicomFolderPath);
|
||||
|
||||
foreach (var file in noneDicomStudy.FileList)
|
||||
{
|
||||
string destinationPath = Path.Combine(studyNoneDicomFolderPath, Path.GetFileName(file.FileName));
|
||||
|
||||
//加入到下载任务里
|
||||
downloadJobs.Add(() => _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath));
|
||||
//下载到当前目录
|
||||
//await _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#region 异步方式处理
|
||||
|
||||
int totalCount = downloadJobs.Count;
|
||||
int downloadedCount = 0;
|
||||
|
||||
foreach (var job in downloadJobs)
|
||||
foreach (var noneDicomStudy in visitItem.NoneDicomStudyList)
|
||||
{
|
||||
try
|
||||
string studyNoneDicomFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}", $"{noneDicomStudy.StudyCode}_{noneDicomStudy.ImageDate.ToString("yyyy-MM-dd")}_{noneDicomStudy.Modality}");
|
||||
|
||||
Directory.CreateDirectory(studyNoneDicomFolderPath);
|
||||
|
||||
foreach (var file in noneDicomStudy.FileList)
|
||||
{
|
||||
await job();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Logger.Error($"下载失败: {ex.Message}");
|
||||
}
|
||||
string destinationPath = Path.Combine(studyNoneDicomFolderPath, Path.GetFileName(file.FileName));
|
||||
|
||||
downloadedCount++;
|
||||
|
||||
// 每处理50个，输出一次进度（或最后一个时也输出）
|
||||
if (downloadedCount % 50 == 0 || downloadedCount == totalCount)
|
||||
{
|
||||
|
||||
Log.Logger.Error($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
|
||||
//加入到下载任务里
|
||||
downloadJobs.Add(new DownloadJob() { Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_NoneDICOM_{destinationPath}", Action = () => _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath) });
|
||||
|
||||
//下载到当前目录
|
||||
//await _oSSService.DownLoadFromOSSAsync(HttpUtility.UrlDecode(file.Path), destinationPath);
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region 多线程测试
|
||||
|
||||
//const int batchSize = 15;
|
||||
//int totalCount = downloadJobs.Count;
|
||||
//int downloadedCount = 0;
|
||||
//建立压缩包
|
||||
string visitFolderPath = Path.Combine(trialFolderPath, $"{visitItem.SubjectCode}_{visitItem.VisitName}");
|
||||
|
||||
//for (int i = 0; i < downloadJobs.Count; i += batchSize)
|
||||
//{
|
||||
// var batch = downloadJobs.Skip(i).Take(batchSize).Select(job => job());
|
||||
downloadJobs.Add(new DownloadJob()
|
||||
{
|
||||
Name = $"{visitItem.SubjectCode}_{visitItem.VisitName}_Zip",
|
||||
|
||||
// await Task.WhenAll(batch);
|
||||
Action = async () =>
|
||||
{
|
||||
string zipPath = visitFolderPath + ".zip";
|
||||
|
||||
// downloadedCount += batch.Count();
|
||||
if (File.Exists(zipPath))
|
||||
{
|
||||
File.Delete(zipPath);
|
||||
}
|
||||
|
||||
// Console.WriteLine($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
|
||||
//}
|
||||
#endregion
|
||||
ZipFile.CreateFromDirectory(
|
||||
visitFolderPath,
|
||||
zipPath,
|
||||
CompressionLevel.Fastest,
|
||||
false);
|
||||
|
||||
Directory.Delete(visitFolderPath, true);
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
});
|
||||
|
||||
//记录日志
|
||||
downloadJobs.Add(new DownloadJob()
|
||||
{
|
||||
Name = $"{visitItem.SubjectCode}_{visitItem.VisitNum}_{visitItem.VisitName}_Finished",
|
||||
Action = () =>
|
||||
{
|
||||
DownloadLogger.Write(
|
||||
logFilePath: logFilePath,
|
||||
subjectCode: visitItem.SubjectCode,
|
||||
visitNum: visitItem.VisitNum,
|
||||
visitName: visitItem.VisitName,
|
||||
message: "Success");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#region 异步方式处理
|
||||
|
||||
int totalCount = downloadJobs.Count;
|
||||
int downloadedCount = 0;
|
||||
|
||||
foreach (var job in downloadJobs)
|
||||
{
|
||||
try
|
||||
{
|
||||
await job.Action();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Logger.Error($"{job.Name}下载失败: {ex.Message}");
|
||||
}
|
||||
|
||||
downloadedCount++;
|
||||
|
||||
// 每处理50个，输出一次进度（或最后一个时也输出）
|
||||
if (downloadedCount % 50 == 0 || downloadedCount == totalCount)
|
||||
{
|
||||
|
||||
Log.Logger.Warning($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
|
||||
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region 多线程测试
|
||||
|
||||
//const int batchSize = 15;
|
||||
//int totalCount = downloadJobs.Count;
|
||||
//int downloadedCount = 0;
|
||||
|
||||
//for (int i = 0; i < downloadJobs.Count; i += batchSize)
|
||||
//{
|
||||
// var batch = downloadJobs.Skip(i).Take(batchSize).Select(job => job());
|
||||
|
||||
// await Task.WhenAll(batch);
|
||||
|
||||
// downloadedCount += batch.Count();
|
||||
|
||||
// Console.WriteLine($"已下载 {downloadedCount} / {totalCount} 个文件，完成 {(downloadedCount * 100.0 / totalCount):F2}%");
|
||||
//}
|
||||
#endregion
|
||||
|
||||
|
||||
return ResponseOutput.Ok();
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue