houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1588 serverless follow ups 2 (#14998)

Browse Source
This commit is contained in:
martmull
2025-10-09 12:56:59 +02:00
committed by GitHub
parent ebc6bbabaf
commit 5252384f14
18 changed files with 3377 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts
+2 -2
View File
@@ -414,10 +414,10 @@ export class ApplicationSyncService {
};
const createdServerlessFunction =
await this.serverlessFunctionV2Service.createOne(
await this.serverlessFunctionV2Service.createOne({
createServerlessFunctionInput,
workspaceId,
);
});
await this.syncDatabaseEventTriggersForServerlessFunction({
serverlessFunctionId: createdServerlessFunction.id,
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/common-layer-name.ts
-1
View File
@@ -1 +0,0 @@
export const COMMON_LAYER_NAME = 'common-layer';
packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts
+1 -6
View File
@@ -27,7 +27,6 @@ import {
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor';
import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
@@ -134,11 +133,7 @@ export class LambdaDriver implements ServerlessDriver {
}
private getLayerName(serverlessFunction: ServerlessFunctionEntity) {
if (isDefined(serverlessFunction?.serverlessFunctionLayer)) {
return serverlessFunction?.serverlessFunctionLayer.checksum;
}
return COMMON_LAYER_NAME;
return serverlessFunction.serverlessFunctionLayer.checksum;
}
private async createLayerIfNotExists(
packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts
+1 -12
View File
@@ -1,15 +1,12 @@
import { promises as fs } from 'fs';
import { join } from 'path';
import { isDefined } from 'twenty-shared/utils';
import {
type ServerlessDriver,
type ServerlessExecuteResult,
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { ConsoleListener } from 'src/engine/core-modules/serverless/drivers/utils/intercept-console';
@@ -34,17 +31,9 @@ export class LocalDriver implements ServerlessDriver {
private getInMemoryLayerFolderPath = (
serverlessFunction: ServerlessFunctionEntity,
) => {
if (!isDefined(serverlessFunction?.serverlessFunctionLayer?.checksum)) {
return join(
SERVERLESS_TMPDIR_FOLDER,
COMMON_LAYER_NAME,
`${serverlessFunction.layerVersion}`,
);
}
return join(
SERVERLESS_TMPDIR_FOLDER,
serverlessFunction.serverlessFunctionLayer?.checksum,
serverlessFunction.serverlessFunctionLayer.checksum,
);
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies.ts
+8 -23
View File
@@ -3,11 +3,8 @@ import { promisify } from 'util';
import { exec } from 'child_process';
import { join } from 'path';
import { isDefined } from 'twenty-shared/utils';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/serverless/drivers/utils/get-layer-dependencies-dir-name';
import type { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
const execPromise = promisify(exec);
@@ -19,29 +16,17 @@ export const copyAndBuildDependencies = async (
recursive: true,
});
if (!isDefined(serverlessFunction.serverlessFunctionLayer)) {
await fs.cp(
getLayerDependenciesDirName(
serverlessFunction.layerVersion || LAST_LAYER_VERSION,
),
buildDirectory,
{
recursive: true,
},
);
} else {
const packageJson = serverlessFunction.serverlessFunctionLayer.packageJson;
const packageJson = serverlessFunction.serverlessFunctionLayer.packageJson;
const yarnLock = serverlessFunction.serverlessFunctionLayer.yarnLock;
const yarnLock = serverlessFunction.serverlessFunctionLayer.yarnLock;
await fs.writeFile(
join(buildDirectory, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf8',
);
await fs.writeFile(
join(buildDirectory, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf8',
);
await fs.writeFile(join(buildDirectory, 'yarn.lock'), yarnLock, 'utf8');
}
await fs.writeFile(join(buildDirectory, 'yarn.lock'), yarnLock, 'utf8');
await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
recursive: true,
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-last-layer-dependencies.ts → packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-last-common-layer-dependencies.ts
-18
View File
@@ -1,10 +1,7 @@
import fs from 'fs/promises';
import { join } from 'path';
import { isDefined } from 'twenty-shared/utils';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/serverless/drivers/utils/get-layer-dependencies-dir-name';
import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
import { type PackageJson } from 'src/engine/core-modules/application/types/application.types';
@@ -24,18 +21,3 @@ export const getLastCommonLayerDependencies = async (
return { packageJson: JSON.parse(packageJson), yarnLock };
};
export const getLayerDependencies = async (
serverlessFunction: ServerlessFunctionEntity,
): Promise<LayerDependencies> => {
if (!isDefined(serverlessFunction.serverlessFunctionLayer)) {
return getLastCommonLayerDependencies(
serverlessFunction.layerVersion ?? undefined,
);
}
return {
packageJson: serverlessFunction.serverlessFunctionLayer.packageJson,
yarnLock: serverlessFunction.serverlessFunctionLayer.yarnLock,
};
};
packages/twenty-server/src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialE
import { ServerlessFunctionLayerEntity } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.entity';
import { CreateServerlessFunctionLayerInput } from 'src/engine/metadata-modules/serverless-function-layer/dtos/create-serverless-function-layer.input';
import { getLastCommonLayerDependencies } from 'src/engine/core-modules/serverless/drivers/utils/get-last-layer-dependencies';
import { getLastCommonLayerDependencies } from 'src/engine/core-modules/serverless/drivers/utils/get-last-common-layer-dependencies';
import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
@Injectable()
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
-5
View File
@@ -13,10 +13,8 @@ import {
IsString,
IsUUID,
} from 'class-validator';
import GraphQLJSON from 'graphql-type-json';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { InputSchema } from 'src/modules/workflow/workflow-builder/workflow-schema/types/input-schema.type';
@ObjectType('ServerlessFunction')
@Authorize({
@@ -60,9 +58,6 @@ export class ServerlessFunctionDTO {
@Field(() => [String], { nullable: false })
publishedVersions: string[];
@Field(() => GraphQLJSON, { nullable: true })
latestVersionInputSchema: InputSchema;
@HideField()
workspaceId: string;
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts
+4 -11
View File
@@ -19,7 +19,6 @@ import { CronTrigger } from 'src/engine/metadata-modules/cron-trigger/entities/c
import { DatabaseEventTrigger } from 'src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity';
import { RouteTrigger } from 'src/engine/metadata-modules/route-trigger/route-trigger.entity';
import { ServerlessFunctionEntityRelationProperties } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { InputSchema } from 'src/modules/workflow/workflow-builder/workflow-schema/types/input-schema.type';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { ServerlessFunctionLayerEntity } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.entity';
@@ -57,9 +56,6 @@ export class ServerlessFunctionEntity
@Column({ nullable: false, type: 'jsonb', default: [] })
publishedVersions: string[];
@Column({ nullable: true, type: 'jsonb' })
latestVersionInputSchema: InputSchema | null;
@Column({ nullable: false, default: ServerlessFunctionRuntime.NODE22 })
runtime: ServerlessFunctionRuntime;
@@ -67,9 +63,6 @@ export class ServerlessFunctionEntity
@Check(`"timeoutSeconds" >= 1 AND "timeoutSeconds" <= 900`)
timeoutSeconds: number;
@Column({ nullable: true, type: 'integer' })
layerVersion: number | null;
@Column({ nullable: false, type: 'uuid' })
workspaceId: string;
@@ -79,16 +72,16 @@ export class ServerlessFunctionEntity
@Column({ nullable: true, type: 'text' })
checksum: string | null;
@Column({ nullable: true, type: 'uuid' })
serverlessFunctionLayerId: string | null;
@Column({ nullable: false, type: 'uuid' })
serverlessFunctionLayerId: string;
@ManyToOne(
() => ServerlessFunctionLayerEntity,
(serverlessFunctionLayer) => serverlessFunctionLayer.serverlessFunctions,
{ nullable: true },
{ nullable: false },
)
@JoinColumn({ name: 'serverlessFunctionLayerId' })
serverlessFunctionLayer: Relation<ServerlessFunctionLayerEntity> | null;
serverlessFunctionLayer: Relation<ServerlessFunctionLayerEntity>;
@ManyToOne(
() => ApplicationEntity,
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts
+1 -1
View File
@@ -11,7 +11,6 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { FeatureFlagGuard } from 'src/engine/guards/feature-flag.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { ExecuteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input';
import { GetServerlessFunctionSourceCodeInput } from 'src/engine/metadata-modules/serverless-function/dtos/get-serverless-function-source-code.input';
import { PublishServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/publish-serverless-function.input';
@@ -22,6 +21,7 @@ import { UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serve
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { serverlessFunctionGraphQLApiExceptionHandler } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
@UseGuards(WorkspaceAuthGuard, FeatureFlagGuard)
@Resolver()
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
+25 -15
View File
@@ -14,12 +14,10 @@ import { AuditService } from 'src/engine/core-modules/audit/services/audit.servi
import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/serverless-function/serverless-function-executed';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { getBaseTypescriptProjectFiles } from 'src/engine/core-modules/serverless/drivers/utils/get-base-typescript-project-files';
import { getLayerDependencies } from 'src/engine/core-modules/serverless/drivers/utils/get-last-layer-dependencies';
import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { type CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { type UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import {
@@ -32,6 +30,7 @@ import {
} from 'src/modules/workflow/common/exceptions/workflow-version-step.exception';
import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service';
import { Sources } from 'src/engine/core-modules/file-storage/types/source.type';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
@Injectable()
export class ServerlessFunctionService {
@@ -291,8 +290,9 @@ export class ServerlessFunctionService {
relations: ['serverlessFunctionLayer'],
});
const { packageJson, yarnLock } =
await getLayerDependencies(serverlessFunction);
const packageJson = serverlessFunction.serverlessFunctionLayer.packageJson;
const yarnLock = serverlessFunction.serverlessFunctionLayer.yarnLock;
const packageVersionRegex = /^"([^@]+)@.*?":\n\s+version: (.+)$/gm;
@@ -314,20 +314,30 @@ export class ServerlessFunctionService {
}
async createOneServerlessFunction(
serverlessFunctionInput: CreateServerlessFunctionInput,
serverlessFunctionInput: CreateServerlessFunctionInput & {
serverlessFunctionLayerId?: string;
},
workspaceId: string,
) {
const commonServerlessFunctionLayer =
await this.serverlessFunctionLayerService.createCommonLayerIfNotExist(
workspaceId,
);
let serverlessFunctionToCreateLayerId =
serverlessFunctionInput.serverlessFunctionLayerId;
if (!isDefined(serverlessFunctionToCreateLayerId)) {
const { id: commonServerlessFunctionLayerId } =
await this.serverlessFunctionLayerService.createCommonLayerIfNotExist(
workspaceId,
);
serverlessFunctionToCreateLayerId = commonServerlessFunctionLayerId;
}
const createServerlessFunctionInput: CreateServerlessFunctionInput = {
...serverlessFunctionInput,
serverlessFunctionLayerId: serverlessFunctionToCreateLayerId,
};
const serverlessFunctionToCreate = this.serverlessFunctionRepository.create(
{
...serverlessFunctionInput,
workspaceId,
serverlessFunctionLayerId: commonServerlessFunctionLayer.id,
},
{ ...createServerlessFunctionInput, workspaceId },
);
const createdServerlessFunction =
@@ -413,7 +423,7 @@ export class ServerlessFunctionService {
timeoutSeconds: serverlessFunctionToDuplicate.timeoutSeconds,
applicationId: serverlessFunctionToDuplicate.applicationId ?? undefined,
serverlessFunctionLayerId:
serverlessFunctionToDuplicate.serverlessFunctionLayerId ?? undefined,
serverlessFunctionToDuplicate.serverlessFunctionLayerId,
},
workspaceId,
);
packages/twenty-server/src/engine/metadata-modules/serverless-function/services/serverless-function-v2.service.ts
+9 -7
View File
@@ -8,7 +8,6 @@ import { deleteFlatEntityFromFlatEntityMapsOrThrow } from 'src/engine/core-modul
import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
import { getSubFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/get-sub-flat-entity-maps-or-throw.util';
import { replaceFlatEntityInFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/replace-flat-entity-in-flat-entity-maps-or-throw.util';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { ServerlessFunctionIdInput } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-id.input';
import { UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input';
import {
@@ -16,7 +15,10 @@ import {
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { fromCreateServerlessFunctionInputToFlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/utils/from-create-serverless-function-input-to-flat-serverless-function.util';
import {
fromCreateServerlessFunctionInputToFlatServerlessFunction,
FromCreateServerlessFunctionInputToFlatServerlessFunctionArgs,
} from 'src/engine/metadata-modules/serverless-function/utils/from-create-serverless-function-input-to-flat-serverless-function.util';
import { fromUpdateServerlessFunctionInputToFlatServerlessFunctionToUpdateOrThrow } from 'src/engine/metadata-modules/serverless-function/utils/from-update-serverless-function-input-to-flat-serverless-function-to-update-or-throw.util';
import { WorkspaceMigrationBuilderExceptionV2 } from 'src/engine/workspace-manager/workspace-migration-v2/exceptions/workspace-migration-builder-exception-v2';
import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-validate-build-and-run-service';
@@ -28,10 +30,10 @@ export class ServerlessFunctionV2Service {
private readonly workspaceMigrationValidateBuildAndRunService: WorkspaceMigrationValidateBuildAndRunService,
) {}
async createOne(
serverlessFunctionInput: CreateServerlessFunctionInput,
workspaceId: string,
) {
async createOne({
createServerlessFunctionInput,
workspaceId,
}: FromCreateServerlessFunctionInputToFlatServerlessFunctionArgs) {
const flatEntityMaps =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
@@ -45,7 +47,7 @@ export class ServerlessFunctionV2Service {
const flatServerlessFunctionToCreate =
fromCreateServerlessFunctionInputToFlatServerlessFunction({
createServerlessFunctionInput: serverlessFunctionInput,
createServerlessFunctionInput,
workspaceId,
});
packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/from-create-serverless-function-input-to-flat-serverless-function.util.ts
+19 -18
View File
@@ -1,44 +1,45 @@
import { v4 } from 'uuid';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
import { type CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { ServerlessFunctionRuntime } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
export const fromCreateServerlessFunctionInputToFlatServerlessFunction = ({
createServerlessFunctionInput,
workspaceId,
}: {
createServerlessFunctionInput: CreateServerlessFunctionInput;
export type FromCreateServerlessFunctionInputToFlatServerlessFunctionArgs = {
createServerlessFunctionInput: CreateServerlessFunctionInput & {
serverlessFunctionLayerId: string;
};
workspaceId: string;
}): FlatServerlessFunction => {
};
export const fromCreateServerlessFunctionInputToFlatServerlessFunction = ({
createServerlessFunctionInput: rawCreateServerlessFunctionInput,
workspaceId,
}: FromCreateServerlessFunctionInputToFlatServerlessFunctionArgs): FlatServerlessFunction => {
const id = v4();
const currentDate = new Date();
return {
id,
name: createServerlessFunctionInput.name,
description: createServerlessFunctionInput.description ?? null,
name: rawCreateServerlessFunctionInput.name,
description: rawCreateServerlessFunctionInput.description ?? null,
universalIdentifier:
createServerlessFunctionInput.universalIdentifier ?? v4(),
rawCreateServerlessFunctionInput.universalIdentifier ?? v4(),
createdAt: currentDate,
updatedAt: currentDate,
deletedAt: null,
latestVersion: null,
publishedVersions: [],
applicationId: createServerlessFunctionInput.applicationId ?? null,
latestVersionInputSchema: null,
applicationId: rawCreateServerlessFunctionInput.applicationId ?? null,
runtime: ServerlessFunctionRuntime.NODE22,
timeoutSeconds: createServerlessFunctionInput.timeoutSeconds ?? 300,
layerVersion: LAST_LAYER_VERSION,
timeoutSeconds: rawCreateServerlessFunctionInput.timeoutSeconds ?? 300,
serverlessFunctionLayerId:
createServerlessFunctionInput.serverlessFunctionLayerId ?? null,
rawCreateServerlessFunctionInput.serverlessFunctionLayerId,
workspaceId,
code: createServerlessFunctionInput?.code,
checksum: createServerlessFunctionInput?.code
code: rawCreateServerlessFunctionInput?.code,
checksum: rawCreateServerlessFunctionInput?.code
? serverlessFunctionCreateHash(
JSON.stringify(createServerlessFunctionInput.code),
JSON.stringify(rawCreateServerlessFunctionInput.code),
)
: null,
};