diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts index 4f8e4e40d3..b02d83e2c7 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts @@ -14,6 +14,7 @@ import { } from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface'; import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type'; +import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service'; import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder'; import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console'; import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager'; @@ -22,10 +23,18 @@ import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic- import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies'; import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service'; import type { SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service'; +import type { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; + +const LAYER_BUILD_LOCK_TTL_MS = 120_000; +const LAYER_BUILD_LOCK_RETRY_MS = 500; +const LAYER_BUILD_LOCK_MAX_RETRIES = 240; +const LAYER_BUILD_READY_SENTINEL = '.twenty-layer-ready'; export interface LocalDriverOptions { logicFunctionResourceService: LogicFunctionResourceService; sdkClientArchiveService: SdkClientArchiveService; + cacheLockService: CacheLockService; + workspaceCacheService: WorkspaceCacheService; } const pathExists = async (targetPath: string): Promise => { @@ -41,10 +50,14 @@ const pathExists = async (targetPath: string): Promise => { export class LocalDriver implements LogicFunctionDriver { private readonly logicFunctionResourceService: LogicFunctionResourceService; private readonly sdkClientArchiveService: SdkClientArchiveService; + private readonly cacheLockService: CacheLockService; + private readonly workspaceCacheService: WorkspaceCacheService; constructor(options: LocalDriverOptions) { this.logicFunctionResourceService = options.logicFunctionResourceService; this.sdkClientArchiveService = options.sdkClientArchiveService; + this.cacheLockService = options.cacheLockService; + this.workspaceCacheService = options.workspaceCacheService; } private getDepsLayerPath(flatApplication: FlatApplication): string { @@ -75,23 +88,40 @@ export class LocalDriver implements LogicFunctionDriver { applicationUniversalIdentifier: string; }): Promise { const depsLayerPath = this.getDepsLayerPath(flatApplication); - const depsNodeModulesPath = join(depsLayerPath, 'node_modules'); + const depsReadySentinelPath = join( + depsLayerPath, + LAYER_BUILD_READY_SENTINEL, + ); - const nodeModulesExist = await pathExists(depsNodeModulesPath); - - if (nodeModulesExist) { + if (await pathExists(depsReadySentinelPath)) { return; } - // Wipe any partial leftovers from a previously failed build - await fs.rm(depsLayerPath, { recursive: true, force: true }); + const lockKey = `local-driver-deps-layer:${flatApplication.yarnLockChecksum ?? 'default'}`; - await this.logicFunctionResourceService.copyDependenciesInMemory({ - applicationUniversalIdentifier, - workspaceId: flatApplication.workspaceId, - inMemoryFolderPath: depsLayerPath, - }); - await copyYarnEngineAndBuildDependencies(depsLayerPath); + await this.cacheLockService.withLock( + async () => { + if (await pathExists(depsReadySentinelPath)) { + return; + } + + await fs.rm(depsLayerPath, { recursive: true, force: true }); + + await this.logicFunctionResourceService.copyDependenciesInMemory({ + applicationUniversalIdentifier, + workspaceId: flatApplication.workspaceId, + inMemoryFolderPath: depsLayerPath, + }); + await copyYarnEngineAndBuildDependencies(depsLayerPath); + await fs.writeFile(depsReadySentinelPath, ''); + }, + lockKey, + { + ttl: LAYER_BUILD_LOCK_TTL_MS, + ms: LAYER_BUILD_LOCK_RETRY_MS, + maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES, + }, + ); } private async ensureSdkLayer({ @@ -106,28 +136,57 @@ export class LocalDriver implements LogicFunctionDriver { applicationUniversalIdentifier, }); const sdkNodeModulesPath = join(sdkLayerPath, 'node_modules'); + const sdkReadySentinelPath = join(sdkLayerPath, LAYER_BUILD_READY_SENTINEL); - const nodeModulesExist = await pathExists(sdkNodeModulesPath); - - if (nodeModulesExist && !flatApplication.isSdkLayerStale) { + if ( + (await pathExists(sdkReadySentinelPath)) && + !flatApplication.isSdkLayerStale + ) { return; } - await fs.rm(sdkLayerPath, { recursive: true, force: true }); + const lockKey = `local-driver-sdk-layer:${flatApplication.workspaceId}:${applicationUniversalIdentifier}`; - const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk'); + await this.cacheLockService.withLock( + async () => { + const { flatApplicationMaps } = + await this.workspaceCacheService.getOrRecompute( + flatApplication.workspaceId, + ['flatApplicationMaps'], + ); + const freshFlatApplication = + flatApplicationMaps.byId[flatApplication.id]; + const isStale = freshFlatApplication?.isSdkLayerStale ?? true; - await this.sdkClientArchiveService.downloadAndExtractToPackage({ - workspaceId: flatApplication.workspaceId, - applicationId: flatApplication.id, - applicationUniversalIdentifier, - targetPackagePath: sdkPackagePath, - }); + if ((await pathExists(sdkReadySentinelPath)) && !isStale) { + return; + } - await this.sdkClientArchiveService.markSdkLayerFresh({ - applicationId: flatApplication.id, - workspaceId: flatApplication.workspaceId, - }); + await fs.rm(sdkLayerPath, { recursive: true, force: true }); + + const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk'); + + await this.sdkClientArchiveService.downloadAndExtractToPackage({ + workspaceId: flatApplication.workspaceId, + applicationId: flatApplication.id, + applicationUniversalIdentifier, + targetPackagePath: sdkPackagePath, + }); + + await this.sdkClientArchiveService.markSdkLayerFresh({ + applicationId: flatApplication.id, + workspaceId: flatApplication.workspaceId, + }); + + await fs.writeFile(sdkReadySentinelPath, ''); + }, + lockKey, + { + ttl: LAYER_BUILD_LOCK_TTL_MS, + ms: LAYER_BUILD_LOCK_RETRY_MS, + maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES, + }, + ); } async transpile({ diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts index 7694e7136d..53f7cd0a02 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts @@ -17,6 +17,7 @@ import { DriverFactoryBase } from 'src/engine/core-modules/twenty-config/dynamic import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { ConfigGroupHashService } from 'src/engine/core-modules/twenty-config/services/config-group-hash.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; @Injectable() export class LogicFunctionDriverFactory extends DriverFactoryBase { @@ -26,6 +27,7 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase