houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload files instead ofsources (#17608)

Browse Source
This commit is contained in:
martmull
2026-02-03 12:11:10 +01:00
committed by GitHub
parent cdaa8f1d9d
commit 65678ed99f
64 changed files with 1148 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/core-modules/application/dtos/application.input.ts
+1 -7
View File
@@ -1,16 +1,10 @@
import { ArgsType, Field } from '@nestjs/graphql';
import GraphQLJSON from 'graphql-type-json';
import { Manifest, PackageJson } from 'twenty-shared/application';
import { Manifest } from 'twenty-shared/application';
@ArgsType()
export class ApplicationInput {
@Field(() => GraphQLJSON, { nullable: false })
manifest: Manifest;
@Field(() => GraphQLJSON, { nullable: false })
packageJson: PackageJson;
@Field(() => String, { nullable: false })
yarnLock: string;
}
packages/twenty-server/src/engine/core-modules/application/dtos/create-application.input.ts
+31
View File
@@ -0,0 +1,31 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
@InputType()
export class CreateApplicationInput {
@IsString()
@IsNotEmpty()
@Field()
universalIdentifier: string;
@IsString()
@IsNotEmpty()
@Field()
name: string;
@IsString()
@IsOptional()
@Field({ nullable: true })
description?: string;
@IsString()
@IsNotEmpty()
@Field()
version: string;
@IsString()
@IsNotEmpty()
@Field()
sourcePath: string;
}
packages/twenty-server/src/engine/core-modules/application/resolvers/application.resolver.ts
+42 -10
View File
@@ -5,12 +5,10 @@ import {
UsePipes,
} from '@nestjs/common';
import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
import { InjectRepository } from '@nestjs/typeorm';
import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
import { PermissionFlagType } from 'twenty-shared/constants';
import { FileFolder } from 'twenty-shared/types';
import { type Repository } from 'typeorm';
import type { FileUpload } from 'graphql-upload/processRequest.mjs';
@@ -30,7 +28,6 @@ import { ApplicationService } from 'src/engine/core-modules/application/services
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { FileDTO } from 'src/engine/core-modules/file/dtos/file.dto';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
@@ -41,6 +38,7 @@ import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/works
import { WorkspaceMigrationGraphqlApiExceptionInterceptor } from 'src/engine/workspace-manager/workspace-migration/interceptors/workspace-migration-graphql-api-exception.interceptor';
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { CreateApplicationInput } from 'src/engine/core-modules/application/dtos/create-application.input';
@UseGuards(
WorkspaceAuthGuard,
@@ -57,8 +55,6 @@ export class ApplicationResolver {
private readonly applicationService: ApplicationService,
private readonly fileStorageService: FileStorageService,
private readonly workspaceCacheService: WorkspaceCacheService,
@InjectRepository(FileEntity)
private readonly fileRepository: Repository<FileEntity>,
) {}
@Query(() => [ApplicationDTO])
@@ -69,31 +65,65 @@ export class ApplicationResolver {
return this.applicationService.findManyApplications(workspaceId);
}
@Query(() => Boolean)
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async checkApplicationExist(
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
@Args('id', { type: () => UUIDScalarType, nullable: true }) id?: string,
@Args('universalIdentifier', { type: () => UUIDScalarType, nullable: true })
universalIdentifier?: string,
) {
return await this.applicationService.checkApplicationExist({
id,
universalIdentifier,
workspaceId,
});
}
@Query(() => ApplicationDTO)
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async findOneApplication(
@Args('id', { type: () => UUIDScalarType }) id: string,
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
@Args('id', { type: () => UUIDScalarType, nullable: true }) id?: string,
@Args('universalIdentifier', { type: () => UUIDScalarType, nullable: true })
universalIdentifier?: string,
) {
return await this.applicationService.findOneApplicationOrThrow({
id,
universalIdentifier,
workspaceId,
});
}
@Mutation(() => ApplicationDTO)
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async createOneApplication(
@Args('input') input: CreateApplicationInput,
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
) {
return await this.applicationService.findOneApplication(id, workspaceId);
return await this.applicationService.create({
...input,
sourceType: 'local',
workspaceId,
});
}
@Mutation(() => Boolean)
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async syncApplication(
@Args() { manifest, packageJson, yarnLock }: ApplicationInput,
@Args() { manifest }: ApplicationInput,
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
) {
await this.applicationSyncService.synchronizeFromManifest({
workspaceId,
manifest,
yarnLock,
packageJson,
});
return true;
}
@Mutation(() => Boolean)
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async installApplication(
@Args() { workspaceMigration: { actions } }: InstallApplicationInput,
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
@@ -132,6 +162,7 @@ export class ApplicationResolver {
}
@Mutation(() => Boolean)
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async uninstallApplication(
@Args() { universalIdentifier }: UninstallApplicationInput,
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
@@ -146,6 +177,7 @@ export class ApplicationResolver {
@Mutation(() => FileDTO)
@UseGuards(SettingsPermissionGuard(PermissionFlagType.UPLOAD_FILE))
@RequireFeatureFlag(FeatureFlagKey.IS_APPLICATION_ENABLED)
async uploadApplicationFile(
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
@Args({ name: 'file', type: () => GraphQLUpload })
packages/twenty-server/src/engine/core-modules/application/services/application-sync.service.ts
+32 -13
View File
@@ -11,8 +11,9 @@ import {
RelationFieldManifest,
RoleManifest,
} from 'twenty-shared/application';
import { FieldMetadataType, Sources } from 'twenty-shared/types';
import { FieldMetadataType, FileFolder, Sources } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { PackageJson } from 'type-fest';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import {
@@ -39,6 +40,8 @@ import { PermissionFlagService } from 'src/engine/metadata-modules/permission-fl
import { RoleService } from 'src/engine/metadata-modules/role/role.service';
import { computeMetadataNameFromLabelOrThrow } from 'src/engine/metadata-modules/utils/compute-metadata-name-from-label-or-throw.util';
import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
@Injectable()
export class ApplicationSyncService {
@@ -58,21 +61,18 @@ export class ApplicationSyncService {
private readonly objectPermissionService: ObjectPermissionService,
private readonly fieldPermissionService: FieldPermissionService,
private readonly permissionService: PermissionFlagService,
private readonly fileStorageService: FileStorageService,
) {}
public async synchronizeFromManifest({
workspaceId,
manifest,
packageJson,
yarnLock,
}: ApplicationInput & {
workspaceId: string;
}) {
const application = await this.syncApplication({
workspaceId,
manifest,
packageJson,
yarnLock,
});
const ownerFlatApplication: FlatApplication = application;
@@ -126,12 +126,24 @@ export class ApplicationSyncService {
private async syncApplication({
workspaceId,
manifest,
packageJson,
yarnLock,
}: ApplicationInput & {
workspaceId: string;
}): Promise<ApplicationEntity> {
const name = manifest.application.displayName ?? packageJson.name;
const name = manifest.application.displayName;
const packageJson = JSON.parse(
(
await streamToBuffer(
await this.fileStorageService.readFile_v2({
applicationUniversalIdentifier:
manifest.application.universalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'package.json',
workspaceId,
}),
)
).toString('utf-8'),
) as PackageJson;
const application =
(await this.applicationService.findByUniversalIdentifier({
universalIdentifier: manifest.application.universalIdentifier,
@@ -150,13 +162,19 @@ export class ApplicationSyncService {
let logicFunctionLayerId = application.logicFunctionLayerId;
if (manifest.logicFunctions.length > 0) {
if (
manifest.logicFunctions.length > 0 &&
isDefined(manifest.application.packageJsonChecksum) &&
isDefined(manifest.application.yarnLockChecksum)
) {
if (!isDefined(logicFunctionLayerId)) {
logicFunctionLayerId = (
await this.logicFunctionLayerService.create(
{
packageJson,
yarnLock,
packageJsonChecksum: manifest.application.packageJsonChecksum,
yarnLockChecksum: manifest.application.yarnLockChecksum,
applicationUniversalIdentifier:
manifest.application.universalIdentifier,
},
workspaceId,
)
@@ -166,9 +184,10 @@ export class ApplicationSyncService {
await this.logicFunctionLayerService.update(
logicFunctionLayerId,
{
packageJson,
yarnLock,
packageJsonChecksum: manifest.application.packageJsonChecksum,
yarnLockChecksum: manifest.application.yarnLockChecksum,
},
manifest.application.universalIdentifier,
workspaceId,
);
}
packages/twenty-server/src/engine/core-modules/application/services/application.service.ts
+60 -7
View File
@@ -123,12 +123,43 @@ export class ApplicationService {
});
}
async findOneApplication(
applicationId: string,
workspaceId: string,
): Promise<ApplicationEntity> {
const application = await this.applicationRepository.findOne({
where: { workspaceId, id: applicationId },
async checkApplicationExist({
id,
universalIdentifier,
workspaceId,
}: {
id?: string;
universalIdentifier?: string;
workspaceId: string;
}) {
return isDefined(
await this.findOneApplication({ id, universalIdentifier, workspaceId }),
);
}
async findOneApplication({
id,
universalIdentifier,
workspaceId,
}: {
id?: string;
universalIdentifier?: string;
workspaceId: string;
}): Promise<ApplicationEntity | null> {
if (!isDefined(id) && !isDefined(universalIdentifier)) {
throw new ApplicationException(
`Either id or universalIdentifier must be provided to find application.`,
ApplicationExceptionCode.APPLICATION_NOT_FOUND,
);
}
const where = {
workspaceId,
...(isDefined(id) ? { id } : { universalIdentifier }),
};
return await this.applicationRepository.findOne({
where,
relations: [
'logicFunctions',
'agents',
@@ -136,10 +167,26 @@ export class ApplicationService {
'applicationVariables',
],
});
}
async findOneApplicationOrThrow({
id,
universalIdentifier,
workspaceId,
}: {
id?: string;
universalIdentifier?: string;
workspaceId: string;
}): Promise<ApplicationEntity> {
const application = await this.findOneApplication({
id,
universalIdentifier,
workspaceId,
});
if (!isDefined(application)) {
throw new ApplicationException(
`Application with id ${applicationId} not found`,
`Application with id ${id} or universalIdentifier ${universalIdentifier} not found`,
ApplicationExceptionCode.APPLICATION_NOT_FOUND,
);
}
@@ -168,6 +215,12 @@ export class ApplicationService {
});
}
async createOneApplication(
data: Partial<ApplicationEntity> & { workspaceId: string },
): Promise<ApplicationEntity> {
return this.create(data);
}
async findTwentyStandardApplicationOrThrow(workspaceId: string): Promise<{
application: ApplicationEntity;
workspace: WorkspaceEntity;
packages/twenty-server/src/engine/core-modules/application/services/marketplace.service.ts
+14 -4
View File
@@ -3,6 +3,7 @@ import { Injectable, Logger } from '@nestjs/common';
// eslint-disable-next-line no-restricted-imports
import { lowerCase, upperFirst } from 'lodash';
import { type Manifest } from 'twenty-shared/application';
import { PackageJson } from 'type-fest';
import { MarketplaceAppDTO } from 'src/engine/core-modules/application/dtos/marketplace-app.dto';
@@ -198,9 +199,18 @@ export class MarketplaceService {
return null;
}
const manifest = JSON.parse(manifestContent) as Manifest;
const packageJsonContent = await this.fetchGitHubFile(
`${appPath}/.twenty/output/package.json`,
);
const { application, packageJson } = manifest;
if (!packageJsonContent) {
return null;
}
const manifest = JSON.parse(manifestContent) as Manifest;
const packageJson = JSON.parse(packageJsonContent) as PackageJson;
const { application } = manifest;
const marketplaceData = application.marketplaceData;
if (!marketplaceData?.author || !marketplaceData?.category) {
@@ -209,10 +219,10 @@ export class MarketplaceService {
return {
id: application.universalIdentifier,
name: application.displayName ?? packageJson.name,
name: application.displayName,
description: application.description ?? '',
icon: application.icon ?? 'IconApps',
version: packageJson.version,
version: packageJson.version ?? '0.1.0',
author: marketplaceData.author,
category: marketplaceData.category,
logo: this.resolveAssetUrl(appPath, marketplaceData.logo),
packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.ts
+5
View File
@@ -17,6 +17,11 @@ export interface StorageDriver {
onStoragePath: string;
}): Promise<void>;
downloadFile(params: {
onStoragePath: string;
localPath: string;
}): Promise<void>;
delete(params: { folderPath: string; filename?: string }): Promise<void>;
move(params: {
from: { folderPath: string; filename?: string };
packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.ts
+13
View File
@@ -72,6 +72,19 @@ export class LocalDriver implements StorageDriver {
await fs.writeFile(filePath, params.sourceFile);
}
async downloadFile(params: {
onStoragePath: string;
localPath: string;
}): Promise<void> {
await this.createFolder(dirname(params.localPath));
const filePath = join(`${this.options.storagePath}/`, params.onStoragePath);
const content = await fs.readFile(filePath);
await fs.writeFile(params.localPath, content);
}
async downloadFolder(params: {
onStoragePath: string;
localPath: string;
packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
+20 -3
View File
@@ -1,8 +1,8 @@
import { Logger } from '@nestjs/common';
import fs from 'fs';
import { mkdir, readdir, readFile } from 'fs/promises';
import { join } from 'path';
import { readdir, readFile } from 'fs/promises';
import { dirname, join } from 'path';
import { Readable } from 'stream';
import { pipeline } from 'stream/promises';
@@ -95,6 +95,23 @@ export class S3Driver implements StorageDriver {
await this.s3Client.send(command);
}
private async createFolder(path: string) {
return fs.mkdirSync(path, { recursive: true });
}
async downloadFile(params: {
onStoragePath: string;
localPath: string;
}): Promise<void> {
await this.createFolder(dirname(params.localPath));
const fileStream = await this.readFile({
filePath: params.onStoragePath,
});
await pipeline(fileStream, fs.createWriteStream(params.localPath));
}
async downloadFolder(params: {
onStoragePath: string;
localPath: string;
@@ -124,7 +141,7 @@ export class S3Driver implements StorageDriver {
? join(params.localPath, relativePath)
: params.localPath;
await mkdir(localFolderPath, { recursive: true });
await this.createFolder(localFolderPath);
const fileStream = await this.readFile({
filePath: `${fromFolderPath}/${filename}`,
packages/twenty-server/src/engine/core-modules/file-storage/file-storage.service.ts
+33 -12
View File
@@ -97,19 +97,28 @@ export class FileStorageService {
sourceFile,
});
const fileEntity = await this.fileRepository.save({
path: `${fileFolder}/${resourcePath}`,
workspaceId,
applicationId: application.id,
id: fileId,
size:
typeof sourceFile === 'string'
? Buffer.byteLength(sourceFile)
: sourceFile.length,
settings,
});
await this.fileRepository.upsert(
{
path: `${fileFolder}/${resourcePath}`,
workspaceId,
applicationId: application.id,
id: fileId,
size:
typeof sourceFile === 'string'
? Buffer.byteLength(sourceFile)
: sourceFile.length,
settings,
},
['path', 'workspaceId', 'applicationId'],
);
return fileEntity;
return await this.fileRepository.findOneOrFail({
where: {
path: `${fileFolder}/${resourcePath}`,
applicationId: application.id,
workspaceId,
},
});
}
/**
@@ -224,6 +233,18 @@ export class FileStorageService {
});
}
downloadFile_v2(
params: ResourceIdentifier & { localPath: string },
): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const onStoragePath = this.buildOnStoragePath(params);
return driver.downloadFile({
onStoragePath,
localPath: params.localPath,
});
}
/**
* @deprecated Use delete_v2 instead
*/
packages/twenty-server/src/engine/core-modules/file/entities/file.entity.ts
+6
View File
@@ -8,6 +8,7 @@ import {
ManyToOne,
PrimaryGeneratedColumn,
Relation,
Unique,
UpdateDateColumn,
} from 'typeorm';
@@ -17,6 +18,11 @@ import { WorkspaceRelatedEntity } from 'src/engine/workspace-manager/types/works
@Entity('file')
@Index('IDX_FILE_WORKSPACE_ID', ['workspaceId'])
@Unique('IDX_APPLICATION_PATH_WORKSPACE_ID_APPLICATION_ID_UNIQUE', [
'workspaceId',
'applicationId',
'path',
])
export class FileEntity extends WorkspaceRelatedEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
packages/twenty-server/src/engine/core-modules/file/files-field/resolvers/files-field.resolver.ts
+1 -3
View File
@@ -34,14 +34,12 @@ export class FilesFieldResolver {
const stream = createReadStream();
const buffer = await streamToBuffer(stream);
const fileEntity = await this.filesFieldService.uploadFile({
return await this.filesFieldService.uploadFile({
file: buffer,
filename,
declaredMimeType: mimetype,
workspaceId,
applicationId: workspaceCustomApplicationId,
});
return fileEntity;
}
}
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-build/services/logic-function-build.service.ts
+65 -1
View File
@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
import fs from 'fs/promises';
import { dirname, join } from 'path';
import crypto from 'crypto';
import { build } from 'esbuild';
import { FileFolder } from 'twenty-shared/types';
@@ -13,6 +14,7 @@ import {
getLogicFunctionBaseFolderPath,
getRelativePathFromBase,
} from 'src/engine/core-modules/logic-function/logic-function-build/utils/get-logic-function-base-folder-path.util';
import { FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
export type FunctionBuildParams = {
flatLogicFunction: FlatLogicFunction;
@@ -23,6 +25,64 @@ export type FunctionBuildParams = {
export class LogicFunctionBuildService {
constructor(private readonly fileStorageService: FileStorageService) {}
async hasLayerDependencies({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}: {
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
}): Promise<boolean> {
const packageJsonExists = await this.fileStorageService.checkFileExists_v2({
workspaceId: flatLogicFunctionLayer.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'package.json',
});
const yarnLockExists = await this.fileStorageService.checkFileExists_v2({
workspaceId: flatLogicFunctionLayer.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'yarn.lock',
});
return packageJsonExists && yarnLockExists;
}
async uploadDependencies({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}: {
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
}) {
await this.fileStorageService.writeFile_v2({
workspaceId: flatLogicFunctionLayer.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'package.json',
sourceFile: JSON.stringify(flatLogicFunctionLayer.packageJson, null, 2),
mimeType: undefined,
settings: {
isTemporaryFile: false,
toDelete: false,
},
});
await this.fileStorageService.writeFile_v2({
workspaceId: flatLogicFunctionLayer.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'yarn.lock',
sourceFile: flatLogicFunctionLayer.yarnLock,
mimeType: undefined,
settings: {
isTemporaryFile: false,
toDelete: false,
},
});
}
async isBuilt({
flatLogicFunction,
applicationUniversalIdentifier,
@@ -38,7 +98,7 @@ export class LogicFunctionBuildService {
async buildAndUpload({
flatLogicFunction,
applicationUniversalIdentifier,
}: FunctionBuildParams): Promise<void> {
}: FunctionBuildParams): Promise<{ checksum: string }> {
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
@@ -85,6 +145,10 @@ export class LogicFunctionBuildService {
toDelete: false,
},
});
return {
checksum: crypto.createHash('md5').update(builtFile).digest('hex'),
};
} finally {
await lambdaBuildDirectoryManager.clean();
}
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/constants/seed-project/src/index.mjs
+8
View File
@@ -0,0 +1,8 @@
var main = async (params) => {
const { a, b } = params;
const message = `Hello, input: ${a} and ${b}`;
return { message };
};
export {
main
};
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
+60 -15
View File
@@ -29,7 +29,7 @@ import {
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-engine-and-build-dependencies';
import { copyExecutor } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-executor';
import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file';
import {
@@ -139,12 +139,43 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
}
private getLayerName(flatLogicFunctionLayer: FlatLogicFunctionLayer) {
return flatLogicFunctionLayer.checksum;
return flatLogicFunctionLayer.yarnLockChecksum;
}
private async createLayerIfNotExists(
flatLogicFunctionLayer: FlatLogicFunctionLayer,
): Promise<string> {
private async copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId,
inMemoryLayerFolderPath,
}: {
applicationUniversalIdentifier: string;
workspaceId: string;
inMemoryLayerFolderPath: string;
}) {
await Promise.all([
this.fileStorageService.downloadFile_v2({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'package.json',
localPath: join(inMemoryLayerFolderPath, 'package.json'),
}),
this.fileStorageService.downloadFile_v2({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'yarn.lock',
localPath: join(inMemoryLayerFolderPath, 'yarn.lock'),
}),
]);
}
private async createLayerIfNotExists({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}: {
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
}): Promise<string> {
const layerName = this.getLayerName(flatLogicFunctionLayer);
const listLayerParams: ListLayerVersionsCommandInput = {
@@ -171,10 +202,12 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
NODE_LAYER_SUBFOLDER,
);
await copyAndBuildDependencies(
nodeDependenciesFolder,
flatLogicFunctionLayer,
);
await this.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatLogicFunctionLayer.workspaceId,
inMemoryLayerFolderPath: nodeDependenciesFolder,
});
await copyYarnEngineAndBuildDependencies(nodeDependenciesFolder);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
@@ -257,15 +290,23 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
return false;
}
private async build(
flatLogicFunction: FlatLogicFunction,
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) {
private async build({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}: {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
}) {
if (await this.isAlreadyBuilt(flatLogicFunction, flatLogicFunctionLayer)) {
return;
}
const layerArn = await this.createLayerIfNotExists(flatLogicFunctionLayer);
const layerArn = await this.createLayerIfNotExists({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
});
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
@@ -317,7 +358,11 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
payload,
env,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build(flatLogicFunction, flatLogicFunctionLayer);
await this.build({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
});
await this.waitFunctionUpdates(flatLogicFunction);
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts
+57 -12
View File
@@ -5,14 +5,14 @@ import { join } from 'path';
import { FileFolder } from 'twenty-shared/types';
import {
type LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionExecutorDriver,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-engine-and-build-dependencies';
import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
@@ -38,13 +38,44 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
) => {
return join(
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
flatLogicFunctionLayer.checksum,
flatLogicFunctionLayer.yarnLockChecksum,
);
};
private async createLayerIfNotExists(
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) {
private async copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId,
inMemoryLayerFolderPath,
}: {
applicationUniversalIdentifier: string;
workspaceId: string;
inMemoryLayerFolderPath: string;
}) {
await Promise.all([
this.fileStorageService.downloadFile_v2({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'package.json',
localPath: join(inMemoryLayerFolderPath, 'package.json'),
}),
this.fileStorageService.downloadFile_v2({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'yarn.lock',
localPath: join(inMemoryLayerFolderPath, 'yarn.lock'),
}),
]);
}
private async createLayerIfNotExists({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}: {
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
}) {
const inMemoryLayerFolderPath = this.getInMemoryLayerFolderPath(
flatLogicFunctionLayer,
);
@@ -52,17 +83,28 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
try {
await fs.access(inMemoryLayerFolderPath);
} catch {
await copyAndBuildDependencies(
await this.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatLogicFunctionLayer.workspaceId,
inMemoryLayerFolderPath,
flatLogicFunctionLayer,
);
});
await copyYarnEngineAndBuildDependencies(inMemoryLayerFolderPath);
}
}
async delete() {}
private async build(flatLogicFunctionLayer: FlatLogicFunctionLayer) {
await this.createLayerIfNotExists(flatLogicFunctionLayer);
private async build({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}: {
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
}) {
await this.createLayerIfNotExists({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
});
}
async execute({
@@ -72,7 +114,10 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
payload,
env,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build(flatLogicFunctionLayer);
await this.build({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
});
const startTime = Date.now();
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies.ts → packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-engine-and-build-dependencies.ts
+1 -15
View File
@@ -4,30 +4,16 @@ import { join } from 'path';
import { promisify } from 'util';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
const execFilePromise = promisify(execFile);
export const copyAndBuildDependencies = async (
export const copyYarnEngineAndBuildDependencies = async (
buildDirectory: string,
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) => {
await fs.mkdir(buildDirectory, {
recursive: true,
});
const packageJson = flatLogicFunctionLayer.packageJson;
const yarnLock = flatLogicFunctionLayer.yarnLock;
await fs.writeFile(
join(buildDirectory, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf8',
);
await fs.writeFile(join(buildDirectory, 'yarn.lock'), yarnLock, 'utf8');
await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
recursive: true,
});
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/utils/get-last-common-layer-dependencies.ts
+2 -4
View File
@@ -1,13 +1,11 @@
import fs from 'fs/promises';
import { join } from 'path';
import { type PackageJson } from 'twenty-shared/application';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version';
export type LayerDependencies = {
packageJson: PackageJson;
packageJson: string;
yarnLock: string;
};
@@ -20,5 +18,5 @@ export const getLastCommonLayerDependencies = async (
fs.readFile(join(lastVersionLayerDirName, 'yarn.lock'), 'utf8'),
]);
return { packageJson: JSON.parse(packageJson), yarnLock };
return { packageJson, yarnLock };
};
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/utils/get-seed-project-files.ts
+1 -1
View File
@@ -16,7 +16,7 @@ const getAllFiles = async (
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
files.push(...(await getAllFiles(rootDir, fullPath, files)));
await getAllFiles(rootDir, fullPath, files);
} else {
files.push({
path: path.relative(rootDir, dir),
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module.ts
+2
View File
@@ -12,6 +12,7 @@ import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.mod
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
import { LogicFunctionLayerModule } from 'src/engine/metadata-modules/logic-function-layer/logic-function-layer.module';
@Module({
imports: [
@@ -22,6 +23,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
SubscriptionsModule,
WorkspaceCacheModule,
LogicFunctionBuildModule,
LogicFunctionLayerModule,
FileModule,
TypeOrmModule.forFeature([LogicFunctionEntity]),
],
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service.ts
+32 -20
View File
@@ -6,7 +6,7 @@ import {
DEFAULT_API_URL_NAME,
} from 'twenty-shared/application';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { isDefined, isEmptyObject } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
import {
@@ -35,6 +35,7 @@ import { SubscriptionChannel } from 'src/engine/subscriptions/enums/subscription
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { cleanServerUrl } from 'src/utils/clean-server-url';
import { LogicFunctionLayerService } from 'src/engine/core-modules/logic-function/logic-function-layer/services/logic-function-layer.service';
const MIN_TOKEN_EXPIRATION_IN_SECONDS = 5;
@@ -68,6 +69,7 @@ export class LogicFunctionExecutorService
private readonly functionBuildService: LogicFunctionBuildService,
private readonly subscriptionService: SubscriptionService,
private readonly auditService: AuditService,
private readonly logicFunctionLayerService: LogicFunctionLayerService,
private readonly fileStorageService: FileStorageService,
@InjectRepository(LogicFunctionEntity)
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
@@ -178,6 +180,19 @@ export class LogicFunctionExecutorService
);
}
// TODO: remove when all logic functions are migrated
if (
!(await this.functionBuildService.hasLayerDependencies({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
}))
) {
await this.functionBuildService.uploadDependencies({
flatLogicFunctionLayer,
applicationUniversalIdentifier,
});
}
if (
!(await this.functionBuildService.isBuilt({
flatLogicFunction,
@@ -189,6 +204,7 @@ export class LogicFunctionExecutorService
applicationUniversalIdentifier,
});
}
// END TODO
const resultLogicFunction = await this.callWithTimeout({
callback: () =>
@@ -291,30 +307,26 @@ export class LogicFunctionExecutorService
async getAvailablePackages(logicFunctionId: string) {
const logicFunction = await this.logicFunctionRepository.findOneOrFail({
where: { id: logicFunctionId },
relations: ['logicFunctionLayer'],
relations: ['logicFunctionLayer', 'application'],
});
const packageJson = logicFunction.logicFunctionLayer.packageJson;
if (isEmptyObject(logicFunction.logicFunctionLayer.availablePackages)) {
await this.logicFunctionLayerService.update(
logicFunction.logicFunctionLayer.id,
{},
logicFunction.application.universalIdentifier,
logicFunction.workspaceId,
);
const yarnLock = logicFunction.logicFunctionLayer.yarnLock;
const packageVersionRegex = /^"([^@]+)@.*?":\n\s+version: (.+)$/gm;
const versions: Record<string, string> = {};
let match: RegExpExecArray | null;
while ((match = packageVersionRegex.exec(yarnLock)) !== null) {
const packageName = match[1].split('@', 1)[0];
const version = match[2];
// @ts-expect-error legacy noImplicitAny
if (packageJson.dependencies?.[packageName]) {
versions[packageName] = version;
}
return (
await this.logicFunctionRepository.findOneOrFail({
where: { id: logicFunctionId },
relations: ['logicFunctionLayer'],
})
).logicFunctionLayer.availablePackages;
}
return versions;
return logicFunction.logicFunctionLayer.availablePackages;
}
private async throttleExecution(workspaceId: string) {
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-layer/services/logic-function-layer.service.ts
+123 -22
View File
@@ -3,6 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { isDefined } from 'twenty-shared/utils';
import { FileFolder } from 'twenty-shared/types';
import { PackageJson } from 'type-fest';
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
@@ -11,6 +13,8 @@ import { CreateLogicFunctionLayerInput } from 'src/engine/metadata-modules/logic
import { getLastCommonLayerDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-last-common-layer-dependencies';
import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
@Injectable()
export class LogicFunctionLayerService {
@@ -18,23 +22,34 @@ export class LogicFunctionLayerService {
@InjectRepository(LogicFunctionLayerEntity)
private readonly logicFunctionLayerRepository: Repository<LogicFunctionLayerEntity>,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly fileStorageService: FileStorageService,
) {}
async create(
{ packageJson, yarnLock }: CreateLogicFunctionLayerInput,
{
packageJsonChecksum,
yarnLockChecksum,
applicationUniversalIdentifier,
}: CreateLogicFunctionLayerInput,
workspaceId: string,
) {
const checksum = logicFunctionCreateHash(yarnLock);
const logicFunctionLayer = this.logicFunctionLayerRepository.create({
packageJson,
yarnLock,
checksum,
packageJson: {}, // TODO: Delete when migration to Source files storage is done
yarnLock: '', // TODO: Delete when migration to Source files storage is done
packageJsonChecksum,
yarnLockChecksum,
workspaceId,
} as Omit<LogicFunctionLayerEntity, 'workspace'>);
const availablePackages = await this.getAvailablePackages({
workspaceId,
applicationUniversalIdentifier,
});
const savedLayer =
await this.logicFunctionLayerRepository.save(logicFunctionLayer);
const savedLayer = await this.logicFunctionLayerRepository.save({
...logicFunctionLayer,
availablePackages,
});
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'logicFunctionLayerMaps',
@@ -45,19 +60,19 @@ export class LogicFunctionLayerService {
async update(
id: string,
data: QueryDeepPartialEntity<LogicFunctionLayerEntity>,
data: QueryDeepPartialEntity<Omit<LogicFunctionLayerEntity, 'workspace'>>,
applicationUniversalIdentifier: string,
workspaceId: string,
) {
const checksum = data.yarnLock
? logicFunctionCreateHash(data.yarnLock as string)
: undefined;
const availablePackages = await this.getAvailablePackages({
workspaceId,
applicationUniversalIdentifier,
});
const updateData = { ...data, ...(checksum && { checksum }) };
const result = await this.logicFunctionLayerRepository.update(
id,
updateData,
);
const result = await this.logicFunctionLayerRepository.update(id, {
...data,
availablePackages,
});
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'logicFunctionLayerMaps',
@@ -66,12 +81,45 @@ export class LogicFunctionLayerService {
return result;
}
async createCommonLayerIfNotExist(workspaceId: string) {
async createCommonLayer({
workspaceId,
applicationUniversalIdentifier,
}: {
workspaceId: string;
applicationUniversalIdentifier: string;
}) {
const { packageJson, yarnLock } = await getLastCommonLayerDependencies();
const checksum = logicFunctionCreateHash(yarnLock);
await this.fileStorageService.writeFile_v2({
sourceFile: packageJson,
mimeType: undefined,
fileFolder: FileFolder.Source,
applicationUniversalIdentifier,
workspaceId,
resourcePath: 'package.json',
settings: { isTemporaryFile: false, toDelete: false },
});
await this.fileStorageService.writeFile_v2({
sourceFile: yarnLock,
mimeType: undefined,
fileFolder: FileFolder.Source,
applicationUniversalIdentifier,
workspaceId,
resourcePath: 'yarn.lock',
settings: { isTemporaryFile: false, toDelete: false },
});
const packageJsonChecksum = logicFunctionCreateHash(
JSON.stringify(packageJson),
);
const yarnLockChecksum = logicFunctionCreateHash(yarnLock);
const commonLayer = await this.logicFunctionLayerRepository.findOne({
where: {
checksum,
yarnLockChecksum,
packageJsonChecksum,
workspaceId,
},
});
@@ -80,6 +128,59 @@ export class LogicFunctionLayerService {
return commonLayer;
}
return this.create({ packageJson, yarnLock }, workspaceId);
return this.create(
{ packageJsonChecksum, yarnLockChecksum, applicationUniversalIdentifier },
workspaceId,
);
}
private async getAvailablePackages({
workspaceId,
applicationUniversalIdentifier,
}: {
workspaceId: string;
applicationUniversalIdentifier: string;
}) {
const packageJson = JSON.parse(
(
await streamToBuffer(
await this.fileStorageService.readFile_v2({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'package.json',
}),
)
).toString('utf-8'),
) as PackageJson;
const yarnLock = (
await streamToBuffer(
await this.fileStorageService.readFile_v2({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: 'yarn.lock',
}),
)
).toString('utf-8');
const packageVersionRegex =
/^"(@?[^@]+(?:\/[^@]+)?)@.*?":\n\s+version:\s*(.+)$/gm;
const versions: Record<string, string> = {};
let match: RegExpExecArray | null;
while ((match = packageVersionRegex.exec(yarnLock)) !== null) {
const packageName = match[1];
const version = match[2];
if (packageJson.dependencies?.[packageName]) {
versions[packageName] = version;
}
}
return versions;
}
}
packages/twenty-server/src/engine/core-modules/public-domain/public-domain.service.ts
+3 -1
View File
@@ -93,7 +93,9 @@ export class PublicDomainService {
try {
await this.publicDomainRepository.insert(
publicDomain as QueryDeepPartialEntity<PublicDomainEntity>,
publicDomain as QueryDeepPartialEntity<
Omit<PublicDomainEntity, 'workspace'>
>,
);
} catch (error) {
await this.dnsManagerService.deleteHostnameSilently(formattedDomain, {