fix: upload chat file_url by API

packages/service/common/file/multer.ts

@@ -16,7 +16,7 @@ export type FileType = {
   size: number;
 };
 
-/* 
+/*
   maxSize: File max size (MB)
 */
 export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {

packages/service/common/s3/sources/dataset/index.ts

@@ -10,8 +10,8 @@ import {
   DeleteDatasetFilesByPrefixParamsSchema,
   type GetDatasetFileContentParams,
   GetDatasetFileContentParamsSchema,
-  type UploadDatasetFileByBufferParams,
-  UploadDatasetFileByBufferParamsSchema
+  type UploadParams,
+  UploadParamsSchema
 } from './type';
 import { MongoS3TTL } from '../../schema';
 import { addHours, addMinutes } from 'date-fns';
@@ -168,23 +168,38 @@ export class S3DatasetSource {
   }
 
   // 根据文件 Buffer 上传文件
-  async uploadDatasetFileByBuffer(params: UploadDatasetFileByBufferParams): Promise<string> {
-    const { datasetId, buffer, filename } = UploadDatasetFileByBufferParamsSchema.parse(params);
+  async uploadDatasetFileByBuffer(params: UploadParams): Promise<string> {
+    const { datasetId, filename, ...file } = UploadParamsSchema.parse(params);
 
-    // 截断文件名以避免S3 key过长的问题
+    // 截断文件名以避免 S3 key 过长的问题
     const truncatedFilename = truncateFilename(filename);
-
     const { fileKey: key } = getFileS3Key.dataset({ datasetId, filename: truncatedFilename });
-    await this.bucket.putObject(key, buffer, buffer.length, {
-      'content-type': Mimes[path.extname(truncatedFilename) as keyof typeof Mimes],
-      'upload-time': new Date().toISOString(),
-      'origin-filename': encodeURIComponent(truncatedFilename)
-    });
+
+    const { stream, size } = (() => {
+      if ('buffer' in file) {
+        return {
+          stream: file.buffer,
+          size: file.buffer.length
+        };
+      }
+      return {
+        stream: file.stream,
+        size: file.size
+      };
+    })();
+
     await MongoS3TTL.create({
       minioKey: key,
       bucketName: this.bucket.name,
       expiredTime: addHours(new Date(), 3)
     });
+
+    await this.bucket.putObject(key, stream, size, {
+      'content-type': Mimes[path.extname(truncatedFilename) as keyof typeof Mimes],
+      'upload-time': new Date().toISOString(),
+      'origin-filename': encodeURIComponent(truncatedFilename)
+    });
+
     return key;
   }
 }

packages/service/common/s3/sources/dataset/type.ts

@@ -1,4 +1,5 @@
 import { ObjectIdSchema } from '@fastgpt/global/common/type/mongo';
+import { ReadStream } from 'node:fs';
 import { z } from 'zod';
 
 export const CreateUploadDatasetFileParamsSchema = z.object({
@@ -44,9 +45,18 @@ export const ParsedFileContentS3KeyParamsSchema = z.object({
 });
 export type ParsedFileContentS3KeyParams = z.infer<typeof ParsedFileContentS3KeyParamsSchema>;
 
-export const UploadDatasetFileByBufferParamsSchema = z.object({
-  datasetId: ObjectIdSchema,
-  buffer: z.instanceof(Buffer),
-  filename: z.string().nonempty()
-});
-export type UploadDatasetFileByBufferParams = z.infer<typeof UploadDatasetFileByBufferParamsSchema>;
+export const UploadParamsSchema = z.union([
+  z.object({
+    datasetId: ObjectIdSchema,
+    filename: z.string().nonempty(),
+    buffer: z.instanceof(Buffer)
+  }),
+
+  z.object({
+    datasetId: ObjectIdSchema,
+    filename: z.string().nonempty(),
+    stream: z.instanceof(ReadStream),
+    size: z.int().positive().optional()
+  })
+]);
+export type UploadParams = z.input<typeof UploadParamsSchema>;

packages/service/common/s3/utils.ts

@@ -11,6 +11,9 @@ import { getNanoid } from '@fastgpt/global/common/string/tools';
 import path from 'node:path';
 import type { ParsedFileContentS3KeyParams } from './sources/dataset/type';
 import { EndpointUrl } from '@fastgpt/global/common/file/constants';
+import m from 'multer';
+import type { NextApiRequest } from 'next';
+import fs from 'node:fs';
 
 // S3文件名最大长度配置
 export const S3_FILENAME_MAX_LENGTH = 50;
@@ -230,3 +233,136 @@ export function isS3ObjectKey<T extends keyof typeof S3Sources>(
 ): key is `${T}/${string}` {
   return typeof key === 'string' && key.startsWith(`${S3Sources[source]}/`);
 }
+
+export const multer = {
+  _storage: m.diskStorage({
+    filename: (_, file, cb) => {
+      if (!file?.originalname) {
+        cb(new Error('File not found'), '');
+      } else {
+        const ext = path.extname(decodeURIComponent(file.originalname));
+        cb(null, `${getNanoid()}${ext}`);
+      }
+    }
+  }),
+
+  singleStore(maxFileSize: number = 500) {
+    const fileSize = maxFileSize * 1024 * 1024;
+
+    return m({
+      limits: {
+        fileSize
+      },
+      preservePath: true,
+      storage: this._storage
+    }).single('file');
+  },
+
+  multipleStore(maxFileSize: number = 500) {
+    const fileSize = maxFileSize * 1024 * 1024;
+
+    return m({
+      limits: {
+        fileSize
+      },
+      preservePath: true,
+      storage: this._storage
+    }).array('file', global.feConfigs?.uploadFileMaxSize);
+  },
+
+  resolveFormData<T extends Record<string, any>>({
+    request,
+    maxFileSize
+  }: {
+    request: NextApiRequest;
+    maxFileSize?: number;
+  }) {
+    return new Promise<{
+      data: T;
+      fileMetadata: Express.Multer.File;
+      getBuffer: () => Buffer;
+      getReadStream: () => fs.ReadStream;
+    }>((resolve, reject) => {
+      const handler = this.singleStore(maxFileSize);
+
+      // @ts-expect-error it can accept a NextApiRequest
+      handler(request, null, (error) => {
+        if (error) {
+          return reject(error);
+        }
+
+        // @ts-expect-error `file` will be injected by multer
+        const file = request.file as Express.Multer.File;
+
+        if (!file) {
+          return reject(new Error('File not found'));
+        }
+
+        const data = (() => {
+          if (!request.body?.data) return {};
+          try {
+            return JSON.parse(request.body.data);
+          } catch {
+            return {};
+          }
+        })();
+
+        resolve({
+          data,
+          fileMetadata: file,
+          getBuffer: () => fs.readFileSync(file.path),
+          getReadStream: () => fs.createReadStream(file.path)
+        });
+      });
+    });
+  },
+
+  resolveMultipleFormData<T extends Record<string, any>>({
+    request,
+    maxFileSize
+  }: {
+    request: NextApiRequest;
+    maxFileSize?: number;
+  }) {
+    return new Promise<{
+      data: T;
+      fileMetadata: Array<Express.Multer.File>;
+    }>((resolve, reject) => {
+      const handler = this.multipleStore(maxFileSize);
+
+      // @ts-expect-error it can accept a NextApiRequest
+      handler(request, null, (error) => {
+        if (error) {
+          return reject(error);
+        }
+
+        // @ts-expect-error `files` will be injected by multer
+        const files = request.files as Array<Express.Multer.File>;
+
+        if (!files || files.length === 0) {
+          return reject(new Error('File not found'));
+        }
+
+        const data = (() => {
+          if (!request.body?.data) return {};
+          try {
+            return JSON.parse(request.body.data);
+          } catch {
+            return {};
+          }
+        })();
+
+        resolve({
+          data,
+          fileMetadata: files
+        });
+      });
+    });
+  },
+
+  clearDiskTempFiles(filepaths: string[]) {
+    for (const filepath of filepaths) {
+      fs.rm(filepath, { force: true }, (_) => {});
+    }
+  }
+};

projects/app/src/pages/api/core/dataset/collection/create/localFile.ts

@@ -1,67 +1,46 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
-import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
-import { getUploadModel } from '@fastgpt/service/common/file/multer';
 import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
-import { type FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
-import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
 import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
-import { getNanoid } from '@fastgpt/global/common/string/tools';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import { NextAPI } from '@/service/middleware/entry';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
 import { type CreateCollectionResponse } from '@/global/core/dataset/api';
+import { multer } from '@fastgpt/service/common/s3/utils';
+import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
 
-async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCollectionResponse {
-  let filePaths: string[] = [];
+async function handler(req: NextApiRequest): CreateCollectionResponse {
+  const filepaths: string[] = [];
 
   try {
-    // Create multer uploader
-    const upload = getUploadModel({
-      maxSize: global.feConfigs?.uploadFileMaxSize
+    const result = await multer.resolveFormData({
+      request: req,
+      maxFileSize: global.feConfigs?.uploadFileMaxSize
     });
-    const { file, data, bucketName } =
-      await upload.getUploadFile<FileCreateDatasetCollectionParams>(
-        req,
-        res,
-        BucketNameEnum.dataset
-      );
-    filePaths = [file.path];
-
-    if (!file || !bucketName) {
-      throw new Error('file is empty');
-    }
+    filepaths.push(result.fileMetadata.path);
 
     const { teamId, tmbId, dataset } = await authDataset({
       req,
       authToken: true,
       authApiKey: true,
       per: WritePermissionVal,
-      datasetId: data.datasetId
+      datasetId: result.data.datasetId
     });
 
-    const { fileMetadata, collectionMetadata, ...collectionData } = data;
-    const collectionName = file.originalname;
+    const { fileMetadata, collectionMetadata, ...collectionData } = result.data;
+    const collectionName = result.fileMetadata.originalname;
 
-    // 1. upload file
-    const fileId = await uploadFile({
-      teamId,
-      uid: tmbId,
-      bucketName,
-      path: file.path,
-      filename: file.originalname,
-      contentType: file.mimetype,
-      metadata: fileMetadata
+    const fileId = await getS3DatasetSource().uploadDatasetFileByBuffer({
+      datasetId: dataset._id,
+      stream: result.getReadStream(),
+      size: result.fileMetadata.size,
+      filename: result.fileMetadata.originalname
     });
 
-    // 2. delete tmp file
-    removeFilesByPaths(filePaths);
-
-    // 3. Create collection
     const { collectionId, insertResults } = await createCollectionAndInsertData({
       dataset,
       createCollectionParams: {
         ...collectionData,
+        datasetId: dataset._id,
         name: collectionName,
         teamId,
         tmbId,
@@ -76,9 +55,9 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
 
     return { collectionId, results: insertResults };
   } catch (error) {
-    removeFilesByPaths(filePaths);
-
     return Promise.reject(error);
+  } finally {
+    multer.clearDiskTempFiles(filepaths);
   }
 }