aws分片上传

2024-09-04 14:50:17 +08:00 · 2024-09-04 14:50:17 +08:00 · 7aaede5503
parent ff7305c3a0
commit 7aaede5503
3 changed files with 117 additions and 91 deletions
--- a/src/utils/multipartUpload/aws.js
+++ b/src/utils/multipartUpload/aws.js
@ -28,7 +28,7 @@ export async function exist(s3, bucket, fileInformation, progressFn, changeStatu
    // 设置每一片的大小,shardSize 指定上传的每个分片的大小，范围为100 KB~5 GB。
    // 分片标准为5MB,文件总大小大于5GB分片为20MB
    let shardSize = 5 * 1024 * 1024;
-    if (uploadFileSize < partSize) {
+    if (uploadFileSize < shardSize) {
        shardSize = uploadFileSize;
    }
    if (uploadFileSize > 5 * 1024 * 1024 * 1024) {
@ -45,8 +45,8 @@ export async function exist(s3, bucket, fileInformation, progressFn, changeStatu
    //判断sharding里面是否有东西，有东西证明已经上传过分片了，不需要再进行检测
    if (fileInformation.sharding.length === 0) {
        let existBucket = await existInBucket({ s3, bucket, fileInformation: fileInformation });
-        console.log("existBucket", existBucket)
        if (existBucket === 'true') {
+            progressFn(0, fileInformation.file, 1);
            changeStatus(fileInformation.path, 'success');//直接告诉前端，状态
            return;
        } else if (existBucket === 'same key') {
@ -183,7 +183,6 @@ async function existInBucket({ s3, bucket, fileInformation }) {
        }
    }
    let bucketFileBufferArray = new Uint8Array(bucketFileUniArray);
-    console.log("bucketFileBufferArray.buffer", bucketFileBufferArray.buffer)
    // 将传入文件的fileReader 转成  arrayBuffer
    let fileArrayBuff = null;
    fileArrayBuff = await new Promise((resolve) => {
@ -287,7 +286,6 @@ async function existUpload({ s3, bucket, fileInformation }) {

 //计算arrayBuffer的md5值
 async function getMD5({ arrayBuffer }) {
-    console.log("arrayBuffer", arrayBuffer)
    return await new Promise((resolve) => {
        const spark = new SparkMD5.ArrayBuffer();
        spark.append(arrayBuffer);
--- a/src/utils/oss.js
+++ b/src/utils/oss.js
@ -16,7 +16,9 @@ Vue.prototype.OSSclientConfig = {

 async function ossGenerateSTS() {
  let res = await GetObjectStoreToken()
+  // res.Result.ObjectStoreUse = 'AWS';
  Vue.prototype.OSSclientConfig = { ...res.Result[res.Result.ObjectStoreUse] }
+  console.log(Vue.prototype.OSSclientConfig);
  Vue.prototype.OSSclientConfig.ObjectStoreUse = res.Result.ObjectStoreUse;
  Vue.prototype.OSSclientConfig.basePath = Vue.prototype.OSSclientConfig.viewEndpoint
  switch (res.Result.ObjectStoreUse) {
@ -26,19 +28,10 @@ async function ossGenerateSTS() {
      Vue.prototype.OSSclientConfig.timeout = 10 * 60 * 1000
      let OSSclient = new OSS(Vue.prototype.OSSclientConfig)
      Vue.prototype.OSSclient = {
-        put: function (objectName, object) {
+        put: async function (objectName, object) {
+          OSSclient = await RefreshClient(OSSclient)
          return new Promise(async (resolve, reject) => {
            try {
-              let config = await getSTSToken(Vue.prototype.OSSclientConfig.Expiration);
-              if (config) {
-                Vue.prototype.OSSclientConfig = { ...config.Result[res.Result.ObjectStoreUse] }
-                Vue.prototype.OSSclientConfig.ObjectStoreUse = config.Result.ObjectStoreUse;
-                Vue.prototype.OSSclientConfig.basePath = Vue.prototype.OSSclientConfig.viewEndpoint;
-                Vue.prototype.OSSclientConfig.bucket = Vue.prototype.OSSclientConfig.bucketName
-                Vue.prototype.OSSclientConfig.stsToken = Vue.prototype.OSSclientConfig.securityToken
-                Vue.prototype.OSSclientConfig.timeout = 10 * 60 * 1000
-                OSSclient = new OSS(Vue.prototype.OSSclientConfig);
-              }
              let _vm = router.default.app
              if (_vm._route.path !== '/trials/trials-panel/visit/crc-upload') {
                var objectItem = objectName.split('/')
@ -60,14 +53,16 @@ async function ossGenerateSTS() {
            }
          })
        },
-        multipartUpload: (data, progress) => {
+        multipartUpload: async (data, progress) => {
+          OSSclient = await RefreshClient(OSSclient)
          return new Promise(async (resolve, reject) => {
            try {
              const { file, path } = data;
              if (!file || !path) return reject('file and path be required');
-              let config = await getSTSToken(Vue.prototype.OSSclientConfig.Expiration);
+              let config = await getSTSToken(Vue.prototype.OSSclientConfig.expiration);
+              console.log(config, 'config')
              if (config) {
-                Vue.prototype.OSSclientConfig = { ...config.Result[res.Result.ObjectStoreUse] }
+                Vue.prototype.OSSclientConfig = { ...config.Result[config.Result.ObjectStoreUse] }
                Vue.prototype.OSSclientConfig.ObjectStoreUse = config.Result.ObjectStoreUse;
                Vue.prototype.OSSclientConfig.basePath = Vue.prototype.OSSclientConfig.viewEndpoint;
                Vue.prototype.OSSclientConfig.bucket = Vue.prototype.OSSclientConfig.bucketName
@ -142,67 +137,57 @@ async function ossGenerateSTS() {
      }
      break
    case 'AWS':
-      let aws = new Minio.Client(Vue.prototype.OSSclientConfig);
+      Vue.prototype.OSSclientConfig.bucket = Vue.prototype.OSSclientConfig.bucketName
+      let aws = new S3Client({
+        endpoint: Vue.prototype.OSSclientConfig.viewEndpoint,
+        region: Vue.prototype.OSSclientConfig.region,
+        s3ForcePathStyle: true,
+        signatureVersion: 'v4',
+        forcePathStyle: true,
+        // SessionToken: '',
+        credentials: {
+          accessKeyId: Vue.prototype.OSSclientConfig.accessKeyId,
+          secretAccessKey: Vue.prototype.OSSclientConfig.secretAccessKey,
+          sessionToken: Vue.prototype.OSSclientConfig.sessionToken
+        }
+      });
      Vue.prototype.OSSclient = {
-        put: function (objectName, object) {
-          return new Promise(async (resolve, reject) => {
-            try {
-              var name = objectName.split('/')[objectName.split('/').length - 1]
-              let _vm = router.default.app
-              if (_vm._route.path !== '/trials/trials-panel/visit/crc-upload') {
-                var objectItem = objectName.split('/')
-                objectItem[objectItem.length - 1] = new Date().getTime() + '_' + objectItem[objectItem.length - 1]
-                objectName = objectItem.join('/')
+        put: async function (objectName, object) {
+          let data = {
+            file: object,
+            path: objectName
          }
-              const reader = new FileReader();
-              reader.onload = (ex) => {
-                const bufferStream = new stream.PassThrough()
-                bufferStream.end(Buffer.from(ex.target.result))
-                aws.putObject(Vue.prototype.OSSclientConfig.bucketName, objectName, bufferStream, function (err, etag) {
-                  if (err) {
-                    console.log(err)
-                    reject()
-                  } else {
-                    console.log(objectName);
-                    resolve({
-                      name: objectName,
-                      url: Vue.prototype.OSSclientConfig.viewEndpoint + decodeUtf8(objectName)
-                    })
-                  }
-                })
-              };
-              reader.readAsArrayBuffer(object);
-            } catch (e) {
-              console.log(e)
-            }
-          })
+          aws = await RefreshClient(aws);
+          return uploadAWS(aws, data, () => { });
        },
-        multipartUpload: (data, progress) => {
+        multipartUpload: async (data, progress) => {
+          aws = await RefreshClient(aws);
+          return uploadAWS(aws, data, progress);
+        },
+        close: () => {
+          AWSclose();
+        }
+      }
+  }
+  return
+}
+// AWS上传函数
+function uploadAWS(aws, data, progress) {
  return new Promise(async (resolve, reject) => {
    try {
      const { file, path } = data;
      if (!file || !path) return reject('file and path be required');
-              let config = await getSTSToken(Vue.prototype.OSSclientConfig.Expiration);
-              if (config) {
-                Vue.prototype.OSSclientConfig = { ...config.Result[res.Result.ObjectStoreUse] }
-                Vue.prototype.OSSclientConfig.ObjectStoreUse = config.Result.ObjectStoreUse;
-                Vue.prototype.OSSclientConfig.basePath = Vue.prototype.OSSclientConfig.viewEndpoint;
-                Vue.prototype.OSSclientConfig.bucket = Vue.prototype.OSSclientConfig.bucketName
-                Vue.prototype.OSSclientConfig.stsToken = Vue.prototype.OSSclientConfig.securityToken
-                Vue.prototype.OSSclientConfig.timeout = 10 * 60 * 1000
-                OSSclient = new S3Client(Vue.prototype.OSSclientConfig);
-              }
      let _vm = router.default.app
      if (_vm._route.path !== '/trials/trials-panel/visit/crc-upload') {
        var objectItem = data.path.split('/')
        objectItem[objectItem.length - 1] = new Date().getTime() + '_' + objectItem[objectItem.length - 1]
        data.path = objectItem.join('/')
      }
-              await exist(OSSclient, Vue.prototype.OSSclientConfig.bucket, data, progress, (res) => {
-                if (res) {
+      await exist(aws, Vue.prototype.OSSclientConfig.bucket, data, progress, (path, status) => {
+        if (status === 'success') {
          resolve({
-                    name: data.path,
-                    url: Vue.prototype.OSSclientConfig.viewEndpoint + res.name
+            name: Vue.prototype.OSSclientConfig.bucket + "/" + decodeUtf8(path),
+            url: Vue.prototype.OSSclientConfig.viewEndpoint + Vue.prototype.OSSclientConfig.bucket + "/" + decodeUtf8(path)
          })
        } else {
          reject()
@ -214,13 +199,48 @@ async function ossGenerateSTS() {
      reject(err)
    }
  })
-        },
-        close: () => {
-          AWSclose();
+}
+// client过期刷新
+async function RefreshClient(client) {
+  let config = await getSTSToken(Vue.prototype.OSSclientConfig.expiration);
+  if (config) {
+    // config.Result.ObjectStoreUse = 'AWS'
+    switch (config.Result.ObjectStoreUse) {
+      case 'AliyunOSS': {
+        Vue.prototype.OSSclientConfig = { ...config.Result[config.Result.ObjectStoreUse] }
+        Vue.prototype.OSSclientConfig.ObjectStoreUse = config.Result.ObjectStoreUse;
+        Vue.prototype.OSSclientConfig.basePath = Vue.prototype.OSSclientConfig.viewEndpoint;
+        Vue.prototype.OSSclientConfig.bucket = Vue.prototype.OSSclientConfig.bucketName
+        Vue.prototype.OSSclientConfig.stsToken = Vue.prototype.OSSclientConfig.securityToken
+        Vue.prototype.OSSclientConfig.timeout = 10 * 60 * 1000
+        return new OSS(Vue.prototype.OSSclientConfig);
+      }
+      case "MinIO": {
+        return client;
+      }
+      case "AWS": {
+        Vue.prototype.OSSclientConfig = { ...config.Result[config.Result.ObjectStoreUse] }
+        Vue.prototype.OSSclientConfig.ObjectStoreUse = config.Result.ObjectStoreUse;
+        Vue.prototype.OSSclientConfig.basePath = Vue.prototype.OSSclientConfig.viewEndpoint;
+        Vue.prototype.OSSclientConfig.bucket = Vue.prototype.OSSclientConfig.bucketName
+        return new S3Client({
+          endpoint: Vue.prototype.OSSclientConfig.viewEndpoint,
+          region: Vue.prototype.OSSclientConfig.region,
+          s3ForcePathStyle: true,
+          signatureVersion: 'v4',
+          forcePathStyle: true,
+          credentials: {
+            accessKeyId: Vue.prototype.OSSclientConfig.accessKeyId,
+            secretAccessKey: Vue.prototype.OSSclientConfig.secretAccessKey,
+            sessionToken: Vue.prototype.OSSclientConfig.sessionToken
+          }
+        });
      }
    }
+
+  } else {
+    return client;
  }
-  return
 }
 function decodeUtf8(bytes) {
  let str = bytes.split('?');
@ -237,7 +257,7 @@ let loading = false;
 function getSTSToken(credentials) {
  return new Promise(async (resolve, reject) => {
    let isExpired = isCredentialsExpired(credentials);
-    if (!isExpired) {
+    if (isExpired) {
      if (loading) {
        queue.push({ resolve, reject })
      }
@ -267,7 +287,7 @@ function isCredentialsExpired(credentials) {
  if (!credentials) {
    return true;
  }
-  const expireDate = new Date(credentials.Expiration);
+  const expireDate = new Date(credentials);
  const now = new Date();
  // 如果有效期不足五分钟，视为过期。
  return expireDate.getTime() - now.getTime() <= 300000;
--- a/src/views/trials/trials-panel/visit/crc-upload/components/uploadDicomFiles2.vue
+++ b/src/views/trials/trials-panel/visit/crc-upload/components/uploadDicomFiles2.vue
@ -885,6 +885,8 @@ export default {
      petVisible: false,
      studyData: [],
      BodyPart: {},
+
+      isClose: false,
    }
  },
  watch: {
@ -908,6 +910,7 @@ export default {
    })
    this.myInterval = []
    store.dispatch('trials/setUnLock', false)
+    this.isClose = true
    this.OSSclient.close()
  },
  methods: {
@ -1678,6 +1681,7 @@ export default {
                                o.instanceUid +
                                params.trialId
                            )}`
+                            if (scope.isClose) return
                            let res = await dcmUpload(
                              {
                                path: path,
@ -1762,7 +1766,10 @@ export default {
                      })
                    })()
                  )
-                  if (arr.length >= 10 || ii === v.instanceList.length - 1) {
+                  if (
+                    (arr.length >= 10 || ii === v.instanceList.length - 1) &&
+                    !scope.isClose
+                  ) {
                    await Promise.all(arr)
                    arr = []
                  }
@ -1834,6 +1841,7 @@ export default {
              if (logRes && logRes.url) {
                params.study.instanceCount = dicomInfo.failedFileCount
                params.RecordPath = scope.$getObjectName(logRes.url)
+                if (scope.isClose) return false
                console.log(params)
                addOrUpdateArchiveStudy(params)
                  .then((res) => {