houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor workflow to use new functions (#17552)

Browse Source 
Removes the versioning system for logic functions (`latestVersion`,
`publishedVersions`) and simplifies the file storage structure.

### Changes
- Remove `publishOneLogicFunctionOrFail` and publishing logic from
workflow status updates
- Add `createLogicFunctionFromExistingLogicFunction` to duplicate logic
functions when creating draft workflow versions
- Update `createDraftStep` to create a new logic function copy instead
of referencing the same one
- Migrate file storage to v2 endpoints with
`applicationUniversalIdentifier`
- Unify path structure: source files at `source/workflow/{id}/`, built
files at `built-logic-function/workflow/{id}/`
- Store full paths in `sourceHandlerPath` and `builtHandlerPath` entity
fields
This commit is contained in:
Charles Bochet
2026-01-30 15:42:36 +01:00
committed by GitHub
parent d624652e36
commit 5996d0fc03
43 changed files with 1213 additions and 1604 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/core-modules/application/application.resolver.ts
+2 -2
View File
@@ -151,7 +151,7 @@ export class ApplicationResolver {
}: UploadApplicationFileInput,
): Promise<FileDTO> {
const allowedApplicationFileFolders: FileFolder[] = [
FileFolder.BuiltFunction,
FileFolder.BuiltLogicFunction,
FileFolder.BuiltFrontComponent,
FileFolder.PublicAsset,
FileFolder.Source,
@@ -178,7 +178,7 @@ export class ApplicationResolver {
dirname,
);
await this.fileStorageService.write({
await this.fileStorageService.writeFile({
file: buffer,
name: filename,
folder: folderPath,
packages/twenty-server/src/engine/core-modules/file-storage/__tests__/file-storage.service.spec.ts
+53 -60
View File
@@ -60,22 +60,21 @@ describe('FileStorageService', () => {
beforeEach(() => {
mockDriver = {
write: jest.fn(),
read: jest.fn(),
writeFile: jest.fn(),
readFile: jest.fn(),
delete: jest.fn(),
move: jest.fn(),
copy: jest.fn(),
download: jest.fn(),
downloadFolder: jest.fn(),
uploadFolder: jest.fn(),
checkFileExists: jest.fn(),
checkFolderExists: jest.fn(),
writeFolder: jest.fn(),
readFolder: jest.fn(),
};
mockFileStorageDriverFactory.getCurrentDriver.mockReturnValue(mockDriver);
});
describe('write', () => {
describe('writeFile', () => {
it('should delegate to the current driver', async () => {
const writeParams = {
file: Buffer.from('test content'),
@@ -84,12 +83,12 @@ describe('FileStorageService', () => {
mimeType: 'text/plain',
};
mockDriver.write.mockResolvedValue(undefined);
mockDriver.writeFile.mockResolvedValue(undefined);
await service.write(writeParams);
await service.writeFile(writeParams);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
expect(mockDriver.write).toHaveBeenCalledWith({
expect(mockDriver.writeFile).toHaveBeenCalledWith({
filePath: 'documents/test.txt',
sourceFile: writeParams.file,
mimeType: 'text/plain',
@@ -106,30 +105,29 @@ describe('FileStorageService', () => {
const error = new Error('Write failed');
mockDriver.write.mockRejectedValue(error);
mockDriver.writeFile.mockRejectedValue(error);
await expect(service.write(writeParams)).rejects.toThrow(
await expect(service.writeFile(writeParams)).rejects.toThrow(
'Write failed',
);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
});
});
describe('read', () => {
describe('readFile', () => {
it('should delegate to the current driver', async () => {
const readParams = {
folderPath: 'documents',
filename: 'test.txt',
filePath: 'documents/test.txt',
};
const mockStream = new Readable();
mockDriver.read.mockResolvedValue(mockStream);
mockDriver.readFile.mockResolvedValue(mockStream);
const result = await service.read(readParams);
const result = await service.readFile(readParams);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
expect(mockDriver.read).toHaveBeenCalledWith({
expect(mockDriver.readFile).toHaveBeenCalledWith({
filePath: 'documents/test.txt',
});
expect(result).toBe(mockStream);
@@ -137,15 +135,16 @@ describe('FileStorageService', () => {
it('should handle read errors', async () => {
const readParams = {
folderPath: 'documents',
filename: 'test.txt',
filePath: 'documents/test.txt',
};
const error = new Error('Read failed');
mockDriver.read.mockRejectedValue(error);
mockDriver.readFile.mockRejectedValue(error);
await expect(service.read(readParams)).rejects.toThrow('Read failed');
await expect(service.readFile(readParams)).rejects.toThrow(
'Read failed',
);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
});
});
@@ -258,44 +257,10 @@ describe('FileStorageService', () => {
});
});
describe('download', () => {
it('should delegate to the current driver', async () => {
const downloadParams = {
from: { folderPath: 'documents', filename: 'test.txt' },
to: { folderPath: '/tmp', filename: 'downloaded-test.txt' },
};
mockDriver.download.mockResolvedValue(undefined);
await service.download(downloadParams);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
expect(mockDriver.download).toHaveBeenCalledWith(downloadParams);
});
it('should handle download errors', async () => {
const downloadParams = {
from: { folderPath: 'documents', filename: 'test.txt' },
to: { folderPath: '/tmp', filename: 'downloaded-test.txt' },
};
const error = new Error('Download failed');
mockDriver.download.mockRejectedValue(error);
await expect(service.download(downloadParams)).rejects.toThrow(
'Download failed',
);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
expect(mockDriver.download).toHaveBeenCalledWith(downloadParams);
});
});
describe('checkFileExists', () => {
it('should delegate to the current driver and return true', async () => {
const checkParams = {
folderPath: 'documents',
filename: 'test.txt',
filePath: 'documents/test.txt',
};
mockDriver.checkFileExists.mockResolvedValue(true);
@@ -309,8 +274,7 @@ describe('FileStorageService', () => {
it('should delegate to the current driver and return false', async () => {
const checkParams = {
folderPath: 'documents',
filename: 'nonexistent.txt',
filePath: 'documents/nonexistent.txt',
};
mockDriver.checkFileExists.mockResolvedValue(false);
@@ -324,8 +288,7 @@ describe('FileStorageService', () => {
it('should handle checkFileExists errors', async () => {
const checkParams = {
folderPath: 'documents',
filename: 'test.txt',
filePath: 'documents/test.txt',
};
const error = new Error('Check failed');
@@ -339,5 +302,35 @@ describe('FileStorageService', () => {
expect(mockDriver.checkFileExists).toHaveBeenCalledWith(checkParams);
});
});
describe('checkFolderExists', () => {
it('should delegate to the current driver and return true', async () => {
const checkParams = {
folderPath: 'documents',
};
mockDriver.checkFolderExists.mockResolvedValue(true);
const result = await service.checkFolderExists(checkParams);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
expect(mockDriver.checkFolderExists).toHaveBeenCalledWith(checkParams);
expect(result).toBe(true);
});
it('should delegate to the current driver and return false', async () => {
const checkParams = {
folderPath: 'nonexistent',
};
mockDriver.checkFolderExists.mockResolvedValue(false);
const result = await service.checkFolderExists(checkParams);
expect(fileStorageDriverFactory.getCurrentDriver).toHaveBeenCalled();
expect(mockDriver.checkFolderExists).toHaveBeenCalledWith(checkParams);
expect(result).toBe(false);
});
});
});
});
packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.ts
+15 -16
View File
@@ -1,17 +1,23 @@
import { type Readable } from 'stream';
import { type Sources } from 'twenty-shared/types';
export interface StorageDriver {
delete(params: { folderPath: string; filename?: string }): Promise<void>;
read(params: { filePath: string }): Promise<Readable>;
readFolder(folderPath: string): Promise<Sources>;
write(params: {
readFile(params: { filePath: string }): Promise<Readable>;
writeFile(params: {
filePath: string;
sourceFile: Buffer | Uint8Array | string;
mimeType: string | undefined;
}): Promise<void>;
writeFolder(sources: Sources, folderPath: string): Promise<void>;
downloadFolder(params: {
onStoragePath: string;
localPath: string;
}): Promise<void>;
uploadFolder(params: {
localPath: string;
onStoragePath: string;
}): Promise<void>;
delete(params: { folderPath: string; filename?: string }): Promise<void>;
move(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
@@ -20,14 +26,7 @@ export interface StorageDriver {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void>;
download(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void>;
checkFileExists(params: {
folderPath: string;
filename: string;
}): Promise<boolean>;
checkFolderExists(folderPath: string): Promise<boolean>;
checkFileExists(params: { filePath: string }): Promise<boolean>;
checkFolderExists(params: { folderPath: string }): Promise<boolean>;
}
packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.ts
+81 -83
View File
@@ -3,9 +3,6 @@ import * as fs from 'fs/promises';
import path, { dirname, join } from 'path';
import { type Readable } from 'stream';
import { isObject } from '@sniptt/guards';
import { type Sources } from 'twenty-shared/types';
import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
import {
FileStorageException,
@@ -23,51 +20,11 @@ export class LocalDriver implements StorageDriver {
this.options = options;
}
async createFolder(path: string) {
private async createFolder(path: string) {
return fs.mkdir(path, { recursive: true });
}
async write(params: {
filePath: string;
sourceFile: Buffer | Uint8Array | string;
mimeType: string | undefined;
}): Promise<void> {
const filePath = `${this.options.storagePath}/${params.filePath}`;
const folderPath = dirname(filePath);
await this.createFolder(folderPath);
await fs.writeFile(filePath, params.sourceFile);
}
async writeFolder(sources: Sources, folderPath: string) {
for (const key of Object.keys(sources)) {
if (isObject(sources[key])) {
await this.writeFolder(sources[key], join(folderPath, key));
continue;
}
await this.write({
filePath: join(folderPath, key),
sourceFile: sources[key],
mimeType: undefined,
});
}
}
async delete(params: {
folderPath: string;
filename?: string;
}): Promise<void> {
const filePath = join(
`${this.options.storagePath}/`,
params.folderPath,
params.filename || '',
);
await fs.rm(filePath, { recursive: true });
}
async read(params: { filePath: string }): Promise<Readable> {
async readFile(params: { filePath: string }): Promise<Readable> {
const joinedPath = join(`${this.options.storagePath}/`, params.filePath);
let filePath: string;
@@ -82,7 +39,6 @@ export class LocalDriver implements StorageDriver {
const storageRoot = realpathSync(path.resolve(this.options.storagePath));
if (!filePath.startsWith(storageRoot + path.sep)) {
// Prevent directory traversal
throw new FileStorageException(
'Access denied',
FileStorageExceptionCode.FILE_NOT_FOUND,
@@ -103,28 +59,87 @@ export class LocalDriver implements StorageDriver {
}
}
async readFolder(folderPath: string): Promise<Sources> {
const sources: Sources = {};
async writeFile(params: {
filePath: string;
sourceFile: Buffer | Uint8Array | string;
mimeType: string | undefined;
}): Promise<void> {
const filePath = `${this.options.storagePath}/${params.filePath}`;
const folderPath = dirname(filePath);
const rootFolderPath = join(`${this.options.storagePath}/`, folderPath);
await this.createFolder(folderPath);
await fs.writeFile(filePath, params.sourceFile);
}
async downloadFolder(params: {
onStoragePath: string;
localPath: string;
}): Promise<void> {
const rootFolderPath = join(
`${this.options.storagePath}/`,
params.onStoragePath,
);
await this.createFolder(params.localPath);
const resources = await fs.readdir(rootFolderPath);
for (const resource of resources) {
const resourcePath = path.join(rootFolderPath, resource);
const stats = await fs.stat(resourcePath);
if (stats.isFile()) {
sources[resource] = await fs.readFile(resourcePath, 'utf8');
const content = await fs.readFile(resourcePath);
await fs.writeFile(path.join(params.localPath, resource), content);
} else {
sources[resource] = await this.readFolder(
path.join(folderPath, resource),
);
await this.downloadFolder({
onStoragePath: path.join(params.onStoragePath, resource),
localPath: path.join(params.localPath, resource),
});
}
}
}
return sources;
async uploadFolder(params: {
localPath: string;
onStoragePath: string;
}): Promise<void> {
const resources = await fs.readdir(params.localPath);
for (const resource of resources) {
const resourcePath = path.join(params.localPath, resource);
const stats = await fs.stat(resourcePath);
if (stats.isFile()) {
const content = await fs.readFile(resourcePath);
await this.writeFile({
filePath: path.join(params.onStoragePath, resource),
sourceFile: content,
mimeType: undefined,
});
} else {
await this.uploadFolder({
localPath: resourcePath,
onStoragePath: path.join(params.onStoragePath, resource),
});
}
}
}
async delete(params: {
folderPath: string;
filename?: string;
}): Promise<void> {
const filePath = join(
`${this.options.storagePath}/`,
params.folderPath,
params.filename || '',
);
await fs.rm(filePath, { recursive: true });
}
async move(params: {
@@ -159,13 +174,10 @@ export class LocalDriver implements StorageDriver {
}
}
async copy(
params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
},
toInMemory = false,
): Promise<void> {
async copy(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
if (!params.from.filename && params.to.filename) {
throw new Error('Cannot copy folder to file');
}
@@ -176,7 +188,7 @@ export class LocalDriver implements StorageDriver {
);
const toPath = join(
toInMemory ? '' : this.options.storagePath,
this.options.storagePath,
params.to.folderPath,
params.to.filename || '',
);
@@ -197,28 +209,14 @@ export class LocalDriver implements StorageDriver {
}
}
async download(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
await this.copy(params, true);
async checkFileExists(params: { filePath: string }): Promise<boolean> {
const fullPath = join(this.options.storagePath, params.filePath);
return existsSync(fullPath);
}
async checkFileExists(params: {
folderPath: string;
filename: string;
}): Promise<boolean> {
const filePath = join(
this.options.storagePath,
params.folderPath,
params.filename,
);
return existsSync(filePath);
}
async checkFolderExists(folderPath: string): Promise<boolean> {
const folderFullPath = join(this.options.storagePath, folderPath);
async checkFolderExists(params: { folderPath: string }): Promise<boolean> {
const folderFullPath = join(this.options.storagePath, params.folderPath);
return existsSync(folderFullPath);
}
packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
+288 -392
View File
@@ -1,7 +1,7 @@
import { Logger } from '@nestjs/common';
import fs from 'fs';
import { mkdir } from 'fs/promises';
import { mkdir, readdir, readFile } from 'fs/promises';
import { join } from 'path';
import { Readable } from 'stream';
import { pipeline } from 'stream/promises';
@@ -20,7 +20,6 @@ import {
S3,
type S3ClientConfig,
} from '@aws-sdk/client-s3';
import { isObject } from '@sniptt/guards';
import { isDefined } from 'twenty-shared/utils';
import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
@@ -29,11 +28,6 @@ import {
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import type { Sources } from 'twenty-shared/types';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { readS3FolderContent } from 'src/engine/core-modules/file-storage/utils/read-s3-folder-content';
export interface S3DriverOptions extends S3ClientConfig {
bucketName: string;
endpoint?: string;
@@ -60,114 +54,7 @@ export class S3Driver implements StorageDriver {
return this.s3Client;
}
async write(params: {
filePath: string;
sourceFile: Buffer | Uint8Array | string;
mimeType: string | undefined;
}): Promise<void> {
const command = new PutObjectCommand({
Key: params.filePath,
Body: params.sourceFile,
ContentType: params.mimeType,
Bucket: this.bucketName,
});
await this.s3Client.send(command);
}
async writeFolder(sources: Sources, folderPath: string) {
for (const key of Object.keys(sources)) {
if (isObject(sources[key])) {
await this.writeFolder(sources[key], join(folderPath, key));
continue;
}
await this.write({
filePath: `${folderPath}/${key}`,
sourceFile: sources[key],
mimeType: undefined,
});
}
}
private async fetchS3FolderContents(folderPath: string) {
const listParams = {
Bucket: this.bucketName,
Prefix: folderPath,
};
const listObjectsCommand = new ListObjectsV2Command(listParams);
const listedObjects = await this.s3Client.send(listObjectsCommand);
return listedObjects;
}
// @ts-expect-error legacy noImplicitAny
private async emptyS3Directory(folderPath) {
this.logger.log(`${folderPath} - emptying folder`);
const listedObjects = await this.fetchS3FolderContents(folderPath);
this.logger.log(
`${folderPath} - listed objects`,
listedObjects.Contents,
listedObjects.IsTruncated,
listedObjects.Contents?.length,
);
if (listedObjects.Contents?.length === 0) return;
const deleteParams = {
Bucket: this.bucketName,
Delete: {
Objects: listedObjects.Contents?.map(({ Key }) => {
return { Key };
}),
},
};
const deleteObjectCommand = new DeleteObjectsCommand(deleteParams);
await this.s3Client.send(deleteObjectCommand);
this.logger.log(`${folderPath} - objects deleted`);
if (listedObjects.IsTruncated) {
this.logger.log(`${folderPath} - folder is truncated`);
await this.emptyS3Directory(folderPath);
}
}
async delete(params: {
folderPath: string;
filename?: string;
}): Promise<void> {
this.logger.log(
`${params.folderPath} - deleting file ${params.filename} from folder ${params.folderPath}`,
);
if (params.filename) {
const deleteCommand = new DeleteObjectCommand({
Key: `${params.folderPath}/${params.filename}`,
Bucket: this.bucketName,
});
await this.s3Client.send(deleteCommand);
} else {
await this.emptyS3Directory(params.folderPath);
this.logger.log(`${params.folderPath} - folder is empty`);
const deleteEmptyFolderCommand = new DeleteObjectCommand({
Key: `${params.folderPath}`,
Bucket: this.bucketName,
});
await this.s3Client.send(deleteEmptyFolderCommand);
}
}
async read(params: { filePath: string }): Promise<Readable> {
async readFile(params: { filePath: string }): Promise<Readable> {
const command = new GetObjectCommand({
Key: params.filePath,
Bucket: this.bucketName,
@@ -193,44 +80,109 @@ export class S3Driver implements StorageDriver {
}
}
async readFolder(folderPath: string): Promise<Sources> {
const sources: Sources = {};
const listedObjects = await this.fetchS3FolderContents(folderPath);
async writeFile(params: {
filePath: string;
sourceFile: Buffer | Uint8Array | string;
mimeType: string | undefined;
}): Promise<void> {
const command = new PutObjectCommand({
Key: params.filePath,
Body: params.sourceFile,
ContentType: params.mimeType,
Bucket: this.bucketName,
});
await this.s3Client.send(command);
}
async downloadFolder(params: {
onStoragePath: string;
localPath: string;
}): Promise<void> {
const listedObjects = await this.fetchS3FolderContents(
params.onStoragePath,
);
if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
return sources;
return;
}
const files = (
await Promise.all(
listedObjects.Contents.map(async (object) => {
if (!object.Key) {
return;
}
for (const object of listedObjects.Contents) {
const folderAndFilePaths = this.extractFolderAndFilePaths(object.Key);
const folderAndFilePaths = this.extractFolderAndFilePaths(object.Key);
if (!isDefined(folderAndFilePaths)) {
continue;
}
if (!isDefined(folderAndFilePaths)) {
return;
}
const { fromFolderPath, filename } = folderAndFilePaths;
const { fromFolderPath, filename } = folderAndFilePaths;
const relativePath = fromFolderPath
.replace(params.onStoragePath + '/', '')
.replace(params.onStoragePath, '');
const fileContent = await readFileContent(
await this.read({ filePath: `${fromFolderPath}/${filename}` }),
);
const localFolderPath = relativePath
? join(params.localPath, relativePath)
: params.localPath;
const formattedObjectKey = object.Key.replace(
folderPath + '/',
'',
).replace(folderPath, '');
await mkdir(localFolderPath, { recursive: true });
return { path: formattedObjectKey, fileContent };
}),
)
).filter(isDefined);
const fileStream = await this.readFile({
filePath: `${fromFolderPath}/${filename}`,
});
return readS3FolderContent(files);
const toPath = join(localFolderPath, filename);
await pipeline(fileStream, fs.createWriteStream(toPath));
}
}
async uploadFolder(params: {
localPath: string;
onStoragePath: string;
}): Promise<void> {
const entries = await readdir(params.localPath, { withFileTypes: true });
for (const entry of entries) {
const localEntryPath = join(params.localPath, entry.name);
if (entry.isDirectory()) {
await this.uploadFolder({
localPath: localEntryPath,
onStoragePath: join(params.onStoragePath, entry.name),
});
} else {
const fileContent = await readFile(localEntryPath);
await this.writeFile({
filePath: `${params.onStoragePath}/${entry.name}`,
sourceFile: fileContent,
mimeType: undefined,
});
}
}
}
async delete(params: {
folderPath: string;
filename?: string;
}): Promise<void> {
if (params.filename) {
const deleteCommand = new DeleteObjectCommand({
Key: `${params.folderPath}/${params.filename}`,
Bucket: this.bucketName,
});
await this.s3Client.send(deleteCommand);
} else {
await this.emptyS3Directory(params.folderPath);
const deleteEmptyFolderCommand = new DeleteObjectCommand({
Key: `${params.folderPath}`,
Bucket: this.bucketName,
});
await this.s3Client.send(deleteEmptyFolderCommand);
}
}
async move(params: {
@@ -247,7 +199,6 @@ export class S3Driver implements StorageDriver {
const toKey = `${params.to.folderPath}/${params.to.filename}`;
try {
// Check if the source file exists
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
@@ -255,7 +206,6 @@ export class S3Driver implements StorageDriver {
}),
);
// Copy the object to the new location
await this.s3Client.send(
new CopyObjectCommand({
CopySource: `${this.bucketName}/${fromKey}`,
@@ -264,7 +214,6 @@ export class S3Driver implements StorageDriver {
}),
);
// Delete the original object
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
@@ -278,12 +227,204 @@ export class S3Driver implements StorageDriver {
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
// For other errors, throw the original error
throw error;
}
}
async moveS3Folder(params: {
async copy(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
if (!params.from.filename && params.to.filename) {
throw new Error('Cannot copy folder to file');
}
const fromKey = `${params.from.folderPath}/${params.from.filename || ''}`;
const toKey = `${params.to.folderPath}/${params.to.filename || ''}`;
if (isDefined(params.from.filename)) {
try {
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
);
await this.s3Client.send(
new CopyObjectCommand({
CopySource: `${this.bucketName}/${fromKey}`,
Bucket: this.bucketName,
Key: toKey,
}),
);
return;
} catch (error) {
if (error.name === 'NotFound') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
throw error;
}
}
const listedObjects = await this.s3Client.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: fromKey,
}),
);
if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
throw new Error(`No objects found in the source folder ${fromKey}.`);
}
for (const object of listedObjects.Contents) {
const folderAndFilePaths = this.extractFolderAndFilePaths(object.Key);
if (!isDefined(folderAndFilePaths)) {
continue;
}
const { fromFolderPath, filename } = folderAndFilePaths;
const toFolderPath = fromFolderPath.replace(
params.from.folderPath,
params.to.folderPath,
);
if (!isDefined(toFolderPath)) {
continue;
}
await this.copy({
from: { folderPath: fromFolderPath, filename },
to: { folderPath: toFolderPath, filename },
});
}
}
async checkFileExists(params: { filePath: string }): Promise<boolean> {
try {
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: params.filePath,
}),
);
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
return true;
}
async checkFolderExists(params: { folderPath: string }): Promise<boolean> {
try {
const listCommand = new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: params.folderPath,
MaxKeys: 1,
});
const result = await this.s3Client.send(listCommand);
return (result.Contents && result.Contents.length > 0) || false;
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
}
async checkBucketExists(args: HeadBucketCommandInput) {
try {
await this.s3Client.headBucket(args);
return true;
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
}
async createBucket(args: CreateBucketCommandInput) {
const exist = await this.checkBucketExists({
Bucket: args.Bucket,
});
if (exist) {
return;
}
return this.s3Client.createBucket(args);
}
private async fetchS3FolderContents(folderPath: string) {
const listParams = {
Bucket: this.bucketName,
Prefix: folderPath,
};
const listObjectsCommand = new ListObjectsV2Command(listParams);
const listedObjects = await this.s3Client.send(listObjectsCommand);
return listedObjects;
}
private async emptyS3Directory(folderPath: string) {
const listedObjects = await this.fetchS3FolderContents(folderPath);
if (listedObjects.Contents?.length === 0) return;
const deleteParams = {
Bucket: this.bucketName,
Delete: {
Objects: listedObjects.Contents?.map(({ Key }) => {
return { Key };
}),
},
};
const deleteObjectCommand = new DeleteObjectsCommand(deleteParams);
await this.s3Client.send(deleteObjectCommand);
if (listedObjects.IsTruncated) {
await this.emptyS3Directory(folderPath);
}
}
private extractFolderAndFilePaths(objectKey: string | undefined) {
if (!isDefined(objectKey)) {
return;
}
const result = /(?<folder>.*)\/(?<file>.*)/.exec(objectKey);
if (!isDefined(result) || !isDefined(result.groups)) {
return;
}
const fromFolderPath = result.groups.folder;
const filename = result.groups.file;
return { fromFolderPath, filename };
}
private async moveS3Folder(params: {
from: { folderPath: string };
to: { folderPath: string };
}): Promise<void> {
@@ -321,249 +462,4 @@ export class S3Driver implements StorageDriver {
});
}
}
extractFolderAndFilePaths(objectKey: string | undefined) {
if (!isDefined(objectKey)) {
return;
}
const result = /(?<folder>.*)\/(?<file>.*)/.exec(objectKey);
if (!isDefined(result) || !isDefined(result.groups)) {
return;
}
const fromFolderPath = result.groups.folder;
const filename = result.groups.file;
return { fromFolderPath, filename };
}
async copy(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
if (!params.from.filename && params.to.filename) {
throw new Error('Cannot copy folder to file');
}
const fromKey = `${params.from.folderPath}/${params.from.filename || ''}`;
const toKey = `${params.to.folderPath}/${params.to.filename || ''}`;
if (isDefined(params.from.filename)) {
try {
// Check if the source file exists
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
);
// Copy the object to the new location
await this.s3Client.send(
new CopyObjectCommand({
CopySource: `${this.bucketName}/${fromKey}`,
Bucket: this.bucketName,
Key: toKey,
}),
);
return;
} catch (error) {
if (error.name === 'NotFound') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
// For other errors, throw the original error
throw error;
}
}
const listedObjects = await this.s3Client.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: fromKey,
}),
);
if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
throw new Error(`No objects found in the source folder ${fromKey}.`);
}
for (const object of listedObjects.Contents) {
const folderAndFilePaths = this.extractFolderAndFilePaths(object.Key);
if (!isDefined(folderAndFilePaths)) {
continue;
}
const { fromFolderPath, filename } = folderAndFilePaths;
const toFolderPath = fromFolderPath.replace(
params.from.folderPath,
params.to.folderPath,
);
if (!isDefined(toFolderPath)) {
continue;
}
await this.copy({
from: {
folderPath: fromFolderPath,
filename,
},
to: { folderPath: toFolderPath, filename },
});
}
}
async download(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
if (!params.from.filename && params.to.filename) {
throw new Error('Cannot copy folder to file');
}
if (isDefined(params.from.filename)) {
try {
const dir = params.to.folderPath;
await mkdir(dir, { recursive: true });
const fileStream = await this.read({
filePath: `${params.from.folderPath}/${params.from.filename}`,
});
const toPath = join(
params.to.folderPath,
params.to.filename || params.from.filename,
);
await pipeline(fileStream, fs.createWriteStream(toPath));
return;
} catch (error) {
if (error.name === 'NotFound') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
// For other errors, throw the original error
throw error;
}
}
const listedObjects = await this.s3Client.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: params.from.folderPath,
}),
);
if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
throw new Error(
`No objects found in the source folder ${params.from.folderPath}.`,
);
}
for (const object of listedObjects.Contents) {
const folderAndFilePaths = this.extractFolderAndFilePaths(object.Key);
if (!isDefined(folderAndFilePaths)) {
continue;
}
const { fromFolderPath, filename } = folderAndFilePaths;
const toFolderPath = fromFolderPath.replace(
params.from.folderPath,
params.to.folderPath,
);
if (!isDefined(toFolderPath)) {
continue;
}
await this.download({
from: {
folderPath: fromFolderPath,
filename,
},
to: { folderPath: toFolderPath, filename },
});
}
}
async checkBucketExists(args: HeadBucketCommandInput) {
try {
await this.s3Client.headBucket(args);
return true;
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
}
async createBucket(args: CreateBucketCommandInput) {
const exist = await this.checkBucketExists({
Bucket: args.Bucket,
});
if (exist) {
return;
}
return this.s3Client.createBucket(args);
}
async checkFileExists(params: {
folderPath: string;
filename: string;
}): Promise<boolean> {
try {
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: `${params.folderPath}/${params.filename}`,
}),
);
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
return true;
}
async checkFolderExists(folderPath: string): Promise<boolean> {
try {
const listCommand = new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: folderPath,
MaxKeys: 1,
});
const result = await this.s3Client.send(listCommand);
return (result.Contents && result.Contents.length > 0) || false;
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
}
}
packages/twenty-server/src/engine/core-modules/file-storage/file-storage.service.ts
+205 -61
View File
@@ -1,8 +1,11 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { mkdir, readdir, readFile, stat } from 'fs/promises';
import { join } from 'path';
import { type Readable } from 'stream';
import { isObject } from '@sniptt/guards';
import { FileFolder, Sources } from 'twenty-shared/types';
import { Like, Repository } from 'typeorm';
@@ -11,9 +14,14 @@ import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/f
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { FileSettings } from 'src/engine/core-modules/file/types/file-settings.types';
export type ResourceIdentifier = {
workspaceId: string;
applicationUniversalIdentifier: string;
fileFolder: FileFolder;
resourcePath: string;
};
@Injectable()
//TODO: Implement storage driver interface when removing v1
//export class FileStorageService implements StorageDriver {
export class FileStorageService {
constructor(
private readonly fileStorageDriverFactory: FileStorageDriverFactory,
@@ -23,10 +31,19 @@ export class FileStorageService {
private readonly applicationRepository: Repository<ApplicationEntity>,
) {}
private buildOnStoragePath({
workspaceId,
applicationUniversalIdentifier,
fileFolder,
resourcePath,
}: ResourceIdentifier): string {
return `${workspaceId}/${applicationUniversalIdentifier}/${fileFolder}/${resourcePath}`;
}
/**
* @deprecated Use write_v2 instead
* @deprecated Use writeFile_v2 instead
*/
write(params: {
writeFile(params: {
file: string | Buffer | Uint8Array;
name: string;
folder: string;
@@ -36,29 +53,25 @@ export class FileStorageService {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.write({
return driver.writeFile({
filePath: `${folder}/${name}`,
sourceFile: file,
mimeType,
});
}
async write_v2({
async writeFile_v2({
sourceFile,
destinationPath,
mimeType,
fileFolder,
applicationUniversalIdentifier,
workspaceId,
resourcePath,
fileId,
settings,
}: {
}: ResourceIdentifier & {
sourceFile: string | Buffer | Uint8Array;
destinationPath: string;
mimeType: string | undefined;
fileFolder: FileFolder;
applicationUniversalIdentifier: string;
workspaceId: string;
fileId?: string;
settings: FileSettings;
}): Promise<FileEntity> {
@@ -71,16 +84,21 @@ export class FileStorageService {
},
});
const driverParams = {
filePath: `${workspaceId}/${applicationUniversalIdentifier}/${fileFolder}/${destinationPath}`,
const onStoragePath = this.buildOnStoragePath({
workspaceId,
applicationUniversalIdentifier,
fileFolder,
resourcePath,
});
await driver.writeFile({
filePath: onStoragePath,
mimeType,
sourceFile,
};
await driver.write(driverParams);
});
const fileEntity = await this.fileRepository.save({
path: `${fileFolder}/${destinationPath}`,
path: `${fileFolder}/${resourcePath}`,
workspaceId,
applicationId: application.id,
id: fileId,
@@ -95,51 +113,133 @@ export class FileStorageService {
}
/**
* @deprecated Use read_v2 instead
* @deprecated Use readFile_v2 instead
*/
read(params: { folderPath: string; filename: string }): Promise<Readable> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const { folderPath, filename } = params;
return driver.read({ filePath: `${folderPath}/${filename}` });
}
read_v2({
destinationPath,
fileFolder,
applicationUniversalIdentifier,
workspaceId,
}: {
destinationPath: string;
fileFolder: FileFolder;
applicationUniversalIdentifier: string;
workspaceId: string;
}): Promise<Readable> {
readFile(params: { filePath: string }): Promise<Readable> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const folderPath = `${workspaceId}/${applicationUniversalIdentifier}/${fileFolder}/${destinationPath}`;
return driver.read({ filePath: folderPath });
return driver.readFile(params);
}
writeFolder(sources: Sources, folderPath: string): Promise<void> {
readFile_v2(params: ResourceIdentifier): Promise<Readable> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.writeFolder(sources, folderPath);
const onStoragePath = this.buildOnStoragePath(params);
return driver.readFile({ filePath: onStoragePath });
}
readFolder(folderPath: string): Promise<Sources> {
/**
* @deprecated Use uploadFolder_v2 with local temp directory instead
*/
async writeFolder(sources: Sources, folderPath: string): Promise<void> {
for (const key of Object.keys(sources)) {
if (isObject(sources[key])) {
await this.writeFolder(sources[key], join(folderPath, key));
continue;
}
await this.writeFile({
file: sources[key],
name: key,
folder: folderPath,
mimeType: undefined,
});
}
}
/**
* @deprecated Use downloadFolder_v2 with local temp directory instead
*/
async readFolder(
folderPath: string,
localTempPath?: string,
): Promise<Sources> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const tempDir = localTempPath || `/tmp/twenty-read-folder-${Date.now()}`;
return driver.readFolder(folderPath);
await mkdir(tempDir, { recursive: true });
await driver.downloadFolder({
onStoragePath: folderPath,
localPath: tempDir,
});
return this.readLocalFolderToSources(tempDir);
}
private async readLocalFolderToSources(localPath: string): Promise<Sources> {
const sources: Sources = {};
const entries = await readdir(localPath);
for (const entry of entries) {
const entryPath = join(localPath, entry);
const stats = await stat(entryPath);
if (stats.isFile()) {
sources[entry] = await readFile(entryPath, 'utf8');
} else {
sources[entry] = await this.readLocalFolderToSources(entryPath);
}
}
return sources;
}
async readFolder_v2(params: ResourceIdentifier): Promise<Sources> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
const tempDir = `/tmp/twenty-read-folder-${Date.now()}`;
await mkdir(tempDir, { recursive: true });
await driver.downloadFolder({
onStoragePath,
localPath: tempDir,
});
return this.readLocalFolderToSources(tempDir);
}
uploadFolder_v2(
params: ResourceIdentifier & { localPath: string },
): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
return driver.uploadFolder({
localPath: params.localPath,
onStoragePath,
});
}
downloadFolder_v2(
params: ResourceIdentifier & { localPath: string },
): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
return driver.downloadFolder({
onStoragePath,
localPath: params.localPath,
});
}
/**
* @deprecated Use delete_v2 instead
*/
delete(params: { folderPath: string; filename?: string }): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.delete(params);
}
delete_v2(params: ResourceIdentifier): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
return driver.delete({ folderPath: onStoragePath });
}
async deleteByFileId({
fileId,
workspaceId,
@@ -166,15 +266,9 @@ export class FileStorageService {
await this.fileRepository.delete(fileId);
}
move(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.move(params);
}
/**
* @deprecated Use copy_v2 instead
*/
copy(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
@@ -184,6 +278,21 @@ export class FileStorageService {
return driver.copy(params);
}
copy_v2({
from,
to,
}: {
from: ResourceIdentifier;
to: ResourceIdentifier;
}): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.copy({
from: { folderPath: this.buildOnStoragePath(from) },
to: { folderPath: this.buildOnStoragePath(to) },
});
}
async moveFile({
from,
to,
@@ -228,27 +337,62 @@ export class FileStorageService {
});
}
download(params: {
/**
* @deprecated Use move_v2 instead
*/
move(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.download(params);
return driver.move(params);
}
checkFileExists(params: {
folderPath: string;
filename: string;
}): Promise<boolean> {
move_v2({
from,
to,
}: {
from: ResourceIdentifier;
to: ResourceIdentifier;
}): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.move({
from: { folderPath: this.buildOnStoragePath(from) },
to: { folderPath: this.buildOnStoragePath(to) },
});
}
/**
* @deprecated Use checkFileExists_v2 instead
*/
checkFileExists(params: { filePath: string }): Promise<boolean> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.checkFileExists(params);
}
checkFolderExists(folderPath: string): Promise<boolean> {
checkFileExists_v2(params: ResourceIdentifier): Promise<boolean> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
return driver.checkFileExists({ filePath: onStoragePath });
}
/**
* @deprecated Use checkFolderExists_v2 instead
*/
checkFolderExists(params: { folderPath: string }): Promise<boolean> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.checkFolderExists(folderPath);
return driver.checkFolderExists(params);
}
checkFolderExists_v2(params: ResourceIdentifier): Promise<boolean> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
return driver.checkFolderExists({ folderPath: onStoragePath });
}
}
packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts
+1 -1
View File
@@ -40,7 +40,7 @@ export class FileUploadService {
mimeType: string | undefined;
folder: string;
}) {
await this.fileStorage.write({
await this.fileStorage.writeFile({
file,
name: filename,
mimeType,
packages/twenty-server/src/engine/core-modules/file/files-field/files-field.service.ts
+4 -4
View File
@@ -69,9 +69,9 @@ export class FilesFieldService {
},
});
return await this.fileStorageService.write_v2({
return await this.fileStorageService.writeFile_v2({
sourceFile: sanitizedFile,
destinationPath: name,
resourcePath: name,
mimeType,
fileFolder: FileFolder.FilesField,
applicationUniversalIdentifier: application.universalIdentifier,
@@ -142,8 +142,8 @@ export class FilesFieldService {
},
});
return await this.fileStorageService.read_v2({
destinationPath: removeFileFolderFromFileEntityPath(file.path),
return await this.fileStorageService.readFile_v2({
resourcePath: removeFileFolderFromFileEntityPath(file.path),
fileFolder: FileFolder.FilesField,
applicationUniversalIdentifier: application.universalIdentifier,
workspaceId,
packages/twenty-server/src/engine/core-modules/file/interfaces/file-folder.interface.ts
+1 -7
View File
@@ -24,19 +24,13 @@ export const fileFolderConfigs: Record<FileFolder, FileFolderConfig> = {
[FileFolder.PersonPicture]: {
ignoreExpirationToken: false,
},
[FileFolder.LogicFunction]: {
ignoreExpirationToken: false,
},
[FileFolder.LogicFunctionToDelete]: {
ignoreExpirationToken: false,
},
[FileFolder.File]: {
ignoreExpirationToken: false,
},
[FileFolder.AgentChat]: {
ignoreExpirationToken: false,
},
[FileFolder.BuiltFunction]: {
[FileFolder.BuiltLogicFunction]: {
ignoreExpirationToken: false,
},
[FileFolder.BuiltFrontComponent]: {
packages/twenty-server/src/engine/core-modules/file/services/file.service.ts
+5 -4
View File
@@ -33,9 +33,8 @@ export class FileService {
): Promise<Readable> {
const workspaceFolderPath = `workspace-${workspaceId}/${folderPath}`;
return await this.fileStorageService.read({
folderPath: workspaceFolderPath,
filename,
return await this.fileStorageService.readFile({
filePath: `${workspaceFolderPath}/${filename}`,
});
}
@@ -96,7 +95,9 @@ export class FileService {
const workspaceFolderPath = `workspace-${workspaceId}`;
const isWorkspaceFolderFound =
await this.fileStorageService.checkFolderExists(workspaceFolderPath);
await this.fileStorageService.checkFolderExists({
folderPath: workspaceFolderPath,
});
if (!isWorkspaceFolderFound) {
return;
packages/twenty-server/src/engine/core-modules/file/utils/__tests__/check-file-path.utils.spec.ts
-35
View File
@@ -1,35 +0,0 @@
import { FileFolder } from 'twenty-shared/types';
import { checkFilePath } from 'src/engine/core-modules/file/utils/check-file-path.utils';
describe('checkFilePath', () => {
it('should return sanitized file path', () => {
const filePath = `${FileFolder.Attachment}\0`;
const sanitizedFilePath = checkFilePath(filePath);
expect(sanitizedFilePath).toBe(`${FileFolder.Attachment}`);
});
it('should return sanitized file path with size', () => {
const filePath = `${FileFolder.ProfilePicture}\0/original`;
const sanitizedFilePath = checkFilePath(filePath);
expect(sanitizedFilePath).toBe(`${FileFolder.ProfilePicture}/original`);
});
it('should throw an error for invalid image size', () => {
const filePath = `${FileFolder.ProfilePicture}\0/invalid-size`;
expect(() => checkFilePath(filePath)).toThrow(
`Size invalid-size is not allowed`,
);
});
it('should throw an error for invalid folder', () => {
const filePath = `invalid-folder`;
expect(() => checkFilePath(filePath)).toThrow(
`Folder invalid-folder is not allowed`,
);
});
});
packages/twenty-server/src/engine/core-modules/file/utils/check-file-path.utils.ts
-32
View File
@@ -1,32 +0,0 @@
import { BadRequestException } from '@nestjs/common';
import { FileFolder } from 'twenty-shared/types';
import { type AllowedFolders } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { kebabCase } from 'src/utils/kebab-case';
import { settings } from 'src/engine/constants/settings';
export const checkFilePath = (filePath: string): string => {
const allowedFolders = Object.values(FileFolder).map((value) =>
kebabCase(value),
);
const sanitizedFilePath = filePath.replace(/\0/g, '');
const [folder, size] = sanitizedFilePath.split('/');
if (!allowedFolders.includes(folder as AllowedFolders)) {
throw new BadRequestException(`Folder ${folder} is not allowed`);
}
if (
folder !== kebabCase(FileFolder.LogicFunction) &&
size &&
// @ts-expect-error legacy noImplicitAny
!settings.storage.imageCropSizes[folder]?.includes(size)
) {
throw new BadRequestException(`Size ${size} is not allowed`);
}
return sanitizedFilePath;
};
packages/twenty-server/src/engine/core-modules/logic-function-executor/drivers/interfaces/logic-function-executor-driver.interface.ts
+11 -11
View File
@@ -16,17 +16,17 @@ export type LogicFunctionExecuteResult = {
error?: LogicFunctionExecuteError;
};
export type LogicFunctionExecuteParams = {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
payload: object;
env?: Record<string, string>;
};
export interface LogicFunctionExecutorDriver {
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
execute({
flatLogicFunction,
flatLogicFunctionLayer,
payload,
env,
}: {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
payload: object;
env?: Record<string, string>;
}): Promise<LogicFunctionExecuteResult>;
execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult>;
}
packages/twenty-server/src/engine/core-modules/logic-function-executor/drivers/lambda.driver.ts
+10 -17
View File
@@ -19,12 +19,13 @@ import {
waitUntilFunctionUpdatedV2,
} from '@aws-sdk/client-lambda';
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
import { isDefined } from 'twenty-shared/utils';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import {
type LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionExecutorDriver,
} from 'src/engine/core-modules/logic-function-executor/drivers/interfaces/logic-function-executor-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
@@ -35,7 +36,6 @@ import {
LambdaBuildDirectoryManager,
NODE_LAYER_SUBFOLDER,
} from 'src/engine/core-modules/logic-function-executor/drivers/utils/lambda-build-directory-manager';
import { getLogicFunctionFolderOrThrow } from 'src/engine/core-modules/logic-function-executor/utils/get-logic-function-folder-or-throw.utils';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
@@ -313,30 +313,23 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
async execute({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
payload,
env,
}: {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
payload: object;
env?: Record<string, string>;
}): Promise<LogicFunctionExecuteResult> {
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build(flatLogicFunction, flatLogicFunctionLayer);
await this.waitFunctionUpdates(flatLogicFunction);
const startTime = Date.now();
const builtHandlerFolderPath = getLogicFunctionFolderOrThrow({
flatLogicFunction,
fileFolder: FileFolder.BuiltFunction,
});
const compiledCode = (
await streamToBuffer(
await this.fileStorageService.read({
folderPath: builtHandlerFolderPath,
filename: flatLogicFunction.builtHandlerPath,
await this.fileStorageService.readFile_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: flatLogicFunction.builtHandlerPath,
}),
)
).toString('utf-8');
packages/twenty-server/src/engine/core-modules/logic-function-executor/drivers/local.driver.ts
+20 -24
View File
@@ -6,6 +6,7 @@ import { FileFolder } from 'twenty-shared/types';
import {
type LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function-executor/drivers/interfaces/logic-function-executor-driver.interface';
@@ -14,10 +15,12 @@ import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/l
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function-executor/drivers/utils/copy-and-build-dependencies';
import { ConsoleListener } from 'src/engine/core-modules/logic-function-executor/drivers/utils/intercept-console';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function-executor/drivers/utils/lambda-build-directory-manager';
import { getLogicFunctionFolderOrThrow } from 'src/engine/core-modules/logic-function-executor/utils/get-logic-function-folder-or-throw.utils';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import {
getLogicFunctionBaseFolderPath,
getRelativePathFromBase,
} from 'src/engine/metadata-modules/logic-function/utils/get-logic-function-base-folder-path.util';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
@@ -65,37 +68,29 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
async execute({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
payload,
env,
}: {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
payload: object;
env?: Record<string, string>;
}): Promise<LogicFunctionExecuteResult> {
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build(flatLogicFunctionLayer);
const startTime = Date.now();
const builtHandlerFolderPath = getLogicFunctionFolderOrThrow({
flatLogicFunction,
fileFolder: FileFolder.BuiltFunction,
});
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
await this.fileStorageService.download({
from: {
folderPath: builtHandlerFolderPath,
filename: flatLogicFunction.builtHandlerPath,
},
to: {
folderPath: sourceTemporaryDir,
filename: flatLogicFunction.builtHandlerPath,
},
const baseFolderPath = getLogicFunctionBaseFolderPath(
flatLogicFunction.builtHandlerPath,
);
await this.fileStorageService.downloadFolder_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: baseFolderPath,
localPath: sourceTemporaryDir,
});
try {
@@ -147,10 +142,11 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
});
try {
const builtBundleFilePath = join(
sourceTemporaryDir,
const relativeBuiltPath = getRelativePathFromBase(
flatLogicFunction.builtHandlerPath,
baseFolderPath,
);
const builtBundleFilePath = join(sourceTemporaryDir, relativeBuiltPath);
const runnerPath = await this.writeBootstrapRunner({
dir: sourceTemporaryDir,
packages/twenty-server/src/engine/core-modules/logic-function-executor/logic-function-executor.service.ts
+5 -18
View File
@@ -2,11 +2,11 @@ import { Inject, Injectable } from '@nestjs/common';
import {
LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function-executor/drivers/interfaces/logic-function-executor-driver.interface';
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function-executor/logic-function-executor.constants';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
@Injectable()
@@ -22,22 +22,9 @@ export class LogicFunctionExecutorService
return this.driver.delete(flatLogicFunction);
}
async execute({
flatLogicFunction,
flatLogicFunctionLayer,
payload,
env,
}: {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
payload: object;
env?: Record<string, string>;
}): Promise<LogicFunctionExecuteResult> {
return this.driver.execute({
flatLogicFunction,
flatLogicFunctionLayer,
payload,
env,
});
async execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult> {
return this.driver.execute(params);
}
}
packages/twenty-server/src/engine/core-modules/logic-function-executor/utils/get-logic-function-folder-or-throw.utils.ts
-22
View File
@@ -1,22 +0,0 @@
import { join } from 'path';
import { FileFolder } from 'twenty-shared/types';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
export const getLogicFunctionFolderOrThrow = ({
flatLogicFunction,
fileFolder = FileFolder.LogicFunction,
}: {
flatLogicFunction: FlatLogicFunction;
fileFolder?:
| FileFolder.LogicFunction
| FileFolder.LogicFunctionToDelete
| FileFolder.BuiltFunction;
}) => {
return join(
'workspace-' + flatLogicFunction.workspaceId,
fileFolder,
flatLogicFunction.id,
);
};
packages/twenty-server/src/engine/core-modules/tool/tools/code-interpreter-tool/code-interpreter-tool.ts
+2 -2
View File
@@ -353,7 +353,7 @@ export class CodeInterpreterTool implements Tool {
const sanitizedFilename = path.basename(file.filename);
try {
await this.fileStorageService.write({
await this.fileStorageService.writeFile({
file: file.content,
name: sanitizedFilename,
mimeType: file.mimeType,
@@ -402,7 +402,7 @@ export class CodeInterpreterTool implements Tool {
}
try {
await this.fileStorageService.write({
await this.fileStorageService.writeFile({
file: file.content,
name: sanitizedFilename,
mimeType: file.mimeType,