From 9983bd65b94433c30ff4741084a0c0e339b7ac45 Mon Sep 17 00:00:00 2001 From: fantasticit Date: Sun, 5 Jun 2022 03:14:47 +0800 Subject: [PATCH] chore: refactor file serve --- packages/client/src/services/file.ts | 31 +--- packages/domains/lib/api/file.d.ts | 8 + packages/domains/lib/api/file.js | 8 + packages/domains/src/api/file.ts | 9 + .../server/src/controllers/file.controller.ts | 10 ++ .../src/helpers/file.helper/aliyun.client.ts | 25 ++- .../src/helpers/file.helper/local.client.ts | 26 ++- .../src/helpers/file.helper/oss.client.ts | 10 +- .../src/helpers/file.helper/tencent.client.ts | 158 ++++++++++-------- packages/server/src/services/file.service.ts | 4 + 10 files changed, 186 insertions(+), 103 deletions(-) diff --git a/packages/client/src/services/file.ts b/packages/client/src/services/file.ts index 9375991..cd2e3a7 100644 --- a/packages/client/src/services/file.ts +++ b/packages/client/src/services/file.ts @@ -83,37 +83,26 @@ export const uploadFile = async ( const unitPercent = 1 / chunks.length; const progressMap = {}; - /** - * 先上传一块分块，如果文件已上传，即无需上传后续分块 - */ - let url = await uploadFileToServer({ - filename, - file: chunks[0], - chunkIndex: 1, - md5, - isChunk: true, - onUploadProgress: (progress) => { - progressMap[1] = progress * unitPercent; - wraponUploadProgress( - Object.keys(progressMap).reduce((a, c) => { - return (a += progressMap[c]); - }, 0) - ); + let url = await HttpClient.request({ + method: FileApiDefinition.initChunk.method, + url: FileApiDefinition.initChunk.client(), + params: { + filename, + md5, }, }); if (!url) { await Promise.all( - chunks.slice(1).map((chunk, index) => { - const currentIndex = 1 + index + 1; + chunks.map((chunk, index) => { return uploadFileToServer({ filename, file: chunk, - chunkIndex: currentIndex, + chunkIndex: index + 1, md5, isChunk: true, onUploadProgress: (progress) => { - progressMap[currentIndex] = progress * unitPercent; + progressMap[index] = progress * unitPercent; wraponUploadProgress( Math.min( Object.keys(progressMap).reduce((a, c) => { @@ -136,9 +125,7 @@ export const uploadFile = async ( }, }); } - wraponUploadProgress(1); - return url; } }; diff --git a/packages/domains/lib/api/file.d.ts b/packages/domains/lib/api/file.d.ts index ca7d397..8759003 100644 --- a/packages/domains/lib/api/file.d.ts +++ b/packages/domains/lib/api/file.d.ts @@ -7,6 +7,14 @@ export declare const FileApiDefinition: { server: "upload"; client: () => string; }; + /** + * 上传分块文件 + */ + initChunk: { + method: "post"; + server: "upload/initChunk"; + client: () => string; + }; /** * 上传分块文件 */ diff --git a/packages/domains/lib/api/file.js b/packages/domains/lib/api/file.js index e802942..14bef71 100644 --- a/packages/domains/lib/api/file.js +++ b/packages/domains/lib/api/file.js @@ -10,6 +10,14 @@ exports.FileApiDefinition = { server: 'upload', client: function () { return '/file/upload'; } }, + /** + * 上传分块文件 + */ + initChunk: { + method: 'post', + server: 'upload/initChunk', + client: function () { return '/file/upload/initChunk'; } + }, /** * 上传分块文件 */ diff --git a/packages/domains/src/api/file.ts b/packages/domains/src/api/file.ts index b724d51..b5c16ae 100644 --- a/packages/domains/src/api/file.ts +++ b/packages/domains/src/api/file.ts @@ -8,6 +8,15 @@ export const FileApiDefinition = { client: () => '/file/upload', }, + /** + * 初始分块上传 + */ + initChunk: { + method: 'post' as const, + server: 'upload/initChunk' as const, + client: () => '/file/upload/initChunk', + }, + /** * 上传分块文件 */ diff --git a/packages/server/src/controllers/file.controller.ts b/packages/server/src/controllers/file.controller.ts index 692f4da..9b3332f 100644 --- a/packages/server/src/controllers/file.controller.ts +++ b/packages/server/src/controllers/file.controller.ts @@ -26,6 +26,16 @@ export class FileController { return this.fileService.uploadFile(file, query); } + /** + * 初始分块文件 + * @param file + */ + @Post(FileApiDefinition.initChunk.server) + @UseGuards(JwtGuard) + initChunk(@Query() query: FileQuery) { + return this.fileService.initChunk(query); + } + /** * 上传分块文件 * @param file diff --git a/packages/server/src/helpers/file.helper/aliyun.client.ts b/packages/server/src/helpers/file.helper/aliyun.client.ts index d092e57..aad1e49 100644 --- a/packages/server/src/helpers/file.helper/aliyun.client.ts +++ b/packages/server/src/helpers/file.helper/aliyun.client.ts @@ -96,18 +96,31 @@ export class AliyunOssClient extends BaseOssClient { * @param query * @returns */ - async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { - const { md5, filename, chunkIndex } = query; - - if (!('chunkIndex' in query)) { - throw new Error('请指定 chunkIndex'); - } + async initChunk(query: FileQuery): Promise { + const { md5, filename } = query; const maybeOssURL = await this.checkIfAlreadyInOss(md5, filename); if (maybeOssURL) { return maybeOssURL; } + return ''; + } + + /** + * 将切片临时存储到服务器 + * FIXME: 阿里云的文档没看懂，故做成这种服务器中转的蠢模式 + * @param file + * @param query + * @returns + */ + async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { + const { md5, chunkIndex } = query; + + if (!('chunkIndex' in query)) { + throw new Error('请指定 chunkIndex'); + } + const dir = this.getStoreDir(md5); const chunksDir = path.join(dir, 'chunks'); fs.ensureDirSync(chunksDir); diff --git a/packages/server/src/helpers/file.helper/local.client.ts b/packages/server/src/helpers/file.helper/local.client.ts index 7becc43..d7a864f 100644 --- a/packages/server/src/helpers/file.helper/local.client.ts +++ b/packages/server/src/helpers/file.helper/local.client.ts @@ -69,16 +69,12 @@ export class LocalOssClient extends BaseOssClient { } /** - * 文件分块上传 + * 文件分块初始化 * @param file * @param query */ - async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { - const { filename, md5, chunkIndex } = query; - - if (!('chunkIndex' in query)) { - throw new Error('请指定 chunkIndex'); - } + async initChunk(query: FileQuery): Promise { + const { filename, md5 } = query; const { absolute, relative } = this.getStoreDir(md5); const absoluteFilepath = path.join(absolute, filename); @@ -88,6 +84,22 @@ export class LocalOssClient extends BaseOssClient { return this.serveFilePath(relativeFilePath); } + return ''; + } + + /** + * 文件分块上传 + * @param file + * @param query + */ + async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { + const { md5, chunkIndex } = query; + + if (!('chunkIndex' in query)) { + throw new Error('请指定 chunkIndex'); + } + + const { absolute } = this.getStoreDir(md5); const chunksDir = path.join(absolute, 'chunks'); fs.ensureDirSync(chunksDir); fs.writeFileSync(path.join(chunksDir, '' + chunkIndex), file.buffer); diff --git a/packages/server/src/helpers/file.helper/oss.client.ts b/packages/server/src/helpers/file.helper/oss.client.ts index 03d99ca..61d0c6f 100644 --- a/packages/server/src/helpers/file.helper/oss.client.ts +++ b/packages/server/src/helpers/file.helper/oss.client.ts @@ -8,7 +8,8 @@ export type FileQuery = { export abstract class OssClient { abstract uploadFile(file: Express.Multer.File, query: FileQuery): Promise; - abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise; + abstract initChunk(query: FileQuery): Promise; + abstract uploadChunk(file: Express.Multer.File, query: FileQuery): Promise; abstract mergeChunk(query: FileQuery): Promise; } @@ -25,7 +26,12 @@ export class BaseOssClient implements OssClient { } // eslint-disable-next-line @typescript-eslint/no-unused-vars - uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { + initChunk(query: FileQuery): Promise { + throw new Error('Method not implemented.'); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { throw new Error('Method not implemented.'); } diff --git a/packages/server/src/helpers/file.helper/tencent.client.ts b/packages/server/src/helpers/file.helper/tencent.client.ts index 10a1d9c..3ecb26d 100644 --- a/packages/server/src/helpers/file.helper/tencent.client.ts +++ b/packages/server/src/helpers/file.helper/tencent.client.ts @@ -1,17 +1,34 @@ import * as TencentCos from 'cos-nodejs-sdk-v5'; +import * as fs from 'fs-extra'; +import * as os from 'os'; +import * as path from 'path'; import { BaseOssClient, FileQuery } from './oss.client'; +/** + * 生产环境会以集群方式运行，通过文件来确保 uploadId 只被初始化一次 + * @param inOssFileName + * @param uploadId + * @returns + */ +function initUploadId(inOssFileName, uploadId) { + const uploadIdFile = path.join(os.tmpdir(), inOssFileName); + fs.ensureFileSync(uploadIdFile); + return fs.writeFileSync(uploadIdFile, uploadId); +} + +function getUploadId(inOssFileName) { + const uploadIdFile = path.join(os.tmpdir(), inOssFileName); + return fs.readFileSync(uploadIdFile, 'utf-8'); +} + +function deleteUploadId(inOssFileName) { + const uploadIdFile = path.join(os.tmpdir(), inOssFileName); + return fs.removeSync(uploadIdFile); +} + export class TencentOssClient extends BaseOssClient { private client: TencentCos | null; - private uploadIdMap: Map = new Map(); - private uploadChunkEtagMap: Map< - string, - { - PartNumber: number; - ETag: string; - }[] - > = new Map(); /** * 构建客户端 @@ -109,36 +126,6 @@ export class TencentOssClient extends BaseOssClient { }); } - /** - * 初始化分块上传 - * @param inOssFileName - * @returns - */ - private getUploadChunkId(inOssFileName): Promise { - if (this.uploadIdMap.has(inOssFileName)) { - return Promise.resolve(this.uploadIdMap.get(inOssFileName)); - } - - return new Promise((resolve, reject) => { - const params = { - Bucket: this.configService.get('oss.tencent.config.Bucket'), - Region: this.configService.get('oss.tencent.config.Region'), - Key: inOssFileName, - }; - this.ensureOssClient(); - this.client.multipartInit(params, (err, data) => { - if (err) { - reject(err); - } else { - const uploadId = data.UploadId; - this.uploadIdMap.set(inOssFileName, uploadId); - this.uploadChunkEtagMap.set(uploadId, []); - resolve(uploadId); - } - }); - }); - } - /** * 上传分片 * @param uploadId @@ -158,14 +145,10 @@ export class TencentOssClient extends BaseOssClient { Body: file.buffer, }; this.ensureOssClient(); - this.client.multipartUpload(params, (err, data) => { + this.client.multipartUpload(params, (err) => { if (err) { reject(err); } else { - this.uploadChunkEtagMap.get(uploadId).push({ - PartNumber: chunkIndex, - ETag: data.ETag, - }); resolve(); } }); @@ -188,26 +171,38 @@ export class TencentOssClient extends BaseOssClient { Key: inOssFileName, }; this.ensureOssClient(); - const parts = this.uploadChunkEtagMap.get(uploadId); - parts.sort((a, b) => a.PartNumber - b.PartNumber); - this.client.multipartComplete( + this.client.multipartListPart( { ...params, UploadId: uploadId, - Parts: parts, }, - (err) => { + (err, data) => { if (err) { reject(err); } else { - this.client.getObjectUrl(params, (err, data) => { - if (err) { - reject(err); - } else { - resolve(data.Url); + const parts = data.Part; + + this.client.multipartComplete( + { + ...params, + UploadId: uploadId, + Parts: parts, + }, + (err) => { + if (err) { + reject(err); + } else { + this.client.getObjectUrl(params, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data.Url); + } + }); + } } - }); + ); } } ); @@ -234,13 +229,52 @@ export class TencentOssClient extends BaseOssClient { return res as string; } + /** + * 初始分片 + * @param file + * @param query + * @returns + */ + async initChunk(query: FileQuery): Promise { + const { md5, filename } = query; + this.ensureOssClient(); + + const inOssFileName = this.getInOssFileName(md5, filename); + const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName); + + if (maybeOssURL) { + return maybeOssURL as string; + } + + const params = { + Bucket: this.configService.get('oss.tencent.config.Bucket'), + Region: this.configService.get('oss.tencent.config.Region'), + Key: inOssFileName, + }; + + const promise = new Promise((resolve, reject) => { + this.client.multipartInit(params, (err, data) => { + if (err) { + reject(err); + } else { + const uploadId = data.UploadId; + initUploadId(inOssFileName, uploadId); + resolve(uploadId); + } + }); + }); + + await promise; + return ''; + } + /** * 上传分片 * @param file * @param query * @returns */ - async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { + async uploadChunk(file: Express.Multer.File, query: FileQuery): Promise { const { md5, filename, chunkIndex } = query; if (!('chunkIndex' in query)) { @@ -249,15 +283,8 @@ export class TencentOssClient extends BaseOssClient { this.ensureOssClient(); const inOssFileName = this.getInOssFileName(md5, filename); - - const maybeOssURL = await this.checkIfAlreadyInOss(inOssFileName); - if (maybeOssURL) { - return maybeOssURL as string; - } - - const uploadId = await this.getUploadChunkId(inOssFileName); + const uploadId = getUploadId(inOssFileName); await this.uploadChunkToCos(uploadId, inOssFileName, chunkIndex, file); - return ''; } /** @@ -268,10 +295,9 @@ export class TencentOssClient extends BaseOssClient { async mergeChunk(query: FileQuery): Promise { const { filename, md5 } = query; const inOssFileName = this.getInOssFileName(md5, filename); - const uploadId = await this.getUploadChunkId(inOssFileName); + const uploadId = getUploadId(inOssFileName); const data = await this.completeUploadChunkToCos(uploadId, inOssFileName); - this.uploadIdMap.delete(inOssFileName); - this.uploadChunkEtagMap.delete(uploadId); + deleteUploadId(inOssFileName); return data; } } diff --git a/packages/server/src/services/file.service.ts b/packages/server/src/services/file.service.ts index 5d0767d..5694acd 100644 --- a/packages/server/src/services/file.service.ts +++ b/packages/server/src/services/file.service.ts @@ -14,6 +14,10 @@ export class FileService { return this.ossClient.uploadFile(file, query); } + async initChunk(query) { + return this.ossClient.initChunk(query); + } + async uploadChunk(file, query) { return this.ossClient.uploadChunk(file, query); }