From 499067ae1429b8fbcb8e0547a8800f7e74cd47b8 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Sun, 26 Apr 2026 14:44:40 +0200 Subject: [PATCH] fix(logic-function): serialize LocalDriver layer builds with cache lock (#20054) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes a race condition in `LocalDriver` where concurrent `execute()` calls for the same application can trash each other's layer build, causing logic function executions to fail with either: - `Error: ENOENT: process.cwd failed with error no such file or directory, the current working directory was likely removed without changing the working directory, uv_cwd` (the yarn-install child's `cwd` got wiped mid-run), or - `ENOENT: no such file or directory, open '…//.js.map'` raised from yarn's berry link step (files under the deps layer vanish while yarn is extracting). Both are thrown from `…/deps//.yarn/releases/yarn-4.9.2.cjs` — i.e. the yarn process `copyYarnEngineAndBuildDependencies` spawns with `cwd: buildDirectory`. ## Root cause `LocalDriver.createLayerIfNotExist` (and `ensureSdkLayer`) both follow a check-then-act pattern with no mutual exclusion: ```ts if (await pathExists(depsNodeModulesPath)) return; await fs.rm(depsLayerPath, { recursive: true, force: true }); await copyDependenciesInMemory(...); await copyYarnEngineAndBuildDependencies(depsLayerPath); // spawns yarn with cwd=depsLayerPath ``` The deps layer path is shared across all `execute()` calls that match a given `yarnLockChecksum`. Two concurrent callers (e.g. a webhook invocation + a cron-triggered logic function firing while the first run's layer is still being built) both see no `node_modules`, both `fs.rm` the directory, and one's yarn child ends up with its `cwd` or its extraction target gone. ## Fix Wrap the critical sections with `CacheLockService.withLock` — the same cache-backed lock the Lambda driver already uses for its layer/executor builds: - `createLayerIfNotExist` → lock key `local-driver-deps-layer:${yarnLockChecksum ?? 'default'}` - `ensureSdkLayer` → lock key `local-driver-sdk-layer:${workspaceId}:${applicationUniversalIdentifier}` A fast-path `pathExists` check is kept outside the lock (so warm executions still skip lock acquisition entirely), and the existence + staleness check is **repeated inside the lock** so followers no-op after the leader finishes. Lock parameters mirror the Lambda driver's layer-build lock (`ttl=120s`, `retry=500ms`, `maxRetries=240`). `cacheLockService` is now passed to `LocalDriver` from `LogicFunctionDriverFactory` (it was already injected there for the Lambda driver). --- .../drivers/local.driver.ts | 113 +++++++++++++----- .../logic-function-driver.factory.ts | 4 + .../logic-function/logic-function.module.ts | 2 + 3 files changed, 92 insertions(+), 27 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts index 4f8e4e40d3..b02d83e2c7 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts @@ -14,6 +14,7 @@ import { } from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface'; import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type'; +import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service'; import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder'; import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console'; import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager'; @@ -22,10 +23,18 @@ import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic- import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies'; import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service'; import type { SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service'; +import type { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; + +const LAYER_BUILD_LOCK_TTL_MS = 120_000; +const LAYER_BUILD_LOCK_RETRY_MS = 500; +const LAYER_BUILD_LOCK_MAX_RETRIES = 240; +const LAYER_BUILD_READY_SENTINEL = '.twenty-layer-ready'; export interface LocalDriverOptions { logicFunctionResourceService: LogicFunctionResourceService; sdkClientArchiveService: SdkClientArchiveService; + cacheLockService: CacheLockService; + workspaceCacheService: WorkspaceCacheService; } const pathExists = async (targetPath: string): Promise => { @@ -41,10 +50,14 @@ const pathExists = async (targetPath: string): Promise => { export class LocalDriver implements LogicFunctionDriver { private readonly logicFunctionResourceService: LogicFunctionResourceService; private readonly sdkClientArchiveService: SdkClientArchiveService; + private readonly cacheLockService: CacheLockService; + private readonly workspaceCacheService: WorkspaceCacheService; constructor(options: LocalDriverOptions) { this.logicFunctionResourceService = options.logicFunctionResourceService; this.sdkClientArchiveService = options.sdkClientArchiveService; + this.cacheLockService = options.cacheLockService; + this.workspaceCacheService = options.workspaceCacheService; } private getDepsLayerPath(flatApplication: FlatApplication): string { @@ -75,23 +88,40 @@ export class LocalDriver implements LogicFunctionDriver { applicationUniversalIdentifier: string; }): Promise { const depsLayerPath = this.getDepsLayerPath(flatApplication); - const depsNodeModulesPath = join(depsLayerPath, 'node_modules'); + const depsReadySentinelPath = join( + depsLayerPath, + LAYER_BUILD_READY_SENTINEL, + ); - const nodeModulesExist = await pathExists(depsNodeModulesPath); - - if (nodeModulesExist) { + if (await pathExists(depsReadySentinelPath)) { return; } - // Wipe any partial leftovers from a previously failed build - await fs.rm(depsLayerPath, { recursive: true, force: true }); + const lockKey = `local-driver-deps-layer:${flatApplication.yarnLockChecksum ?? 'default'}`; - await this.logicFunctionResourceService.copyDependenciesInMemory({ - applicationUniversalIdentifier, - workspaceId: flatApplication.workspaceId, - inMemoryFolderPath: depsLayerPath, - }); - await copyYarnEngineAndBuildDependencies(depsLayerPath); + await this.cacheLockService.withLock( + async () => { + if (await pathExists(depsReadySentinelPath)) { + return; + } + + await fs.rm(depsLayerPath, { recursive: true, force: true }); + + await this.logicFunctionResourceService.copyDependenciesInMemory({ + applicationUniversalIdentifier, + workspaceId: flatApplication.workspaceId, + inMemoryFolderPath: depsLayerPath, + }); + await copyYarnEngineAndBuildDependencies(depsLayerPath); + await fs.writeFile(depsReadySentinelPath, ''); + }, + lockKey, + { + ttl: LAYER_BUILD_LOCK_TTL_MS, + ms: LAYER_BUILD_LOCK_RETRY_MS, + maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES, + }, + ); } private async ensureSdkLayer({ @@ -106,28 +136,57 @@ export class LocalDriver implements LogicFunctionDriver { applicationUniversalIdentifier, }); const sdkNodeModulesPath = join(sdkLayerPath, 'node_modules'); + const sdkReadySentinelPath = join(sdkLayerPath, LAYER_BUILD_READY_SENTINEL); - const nodeModulesExist = await pathExists(sdkNodeModulesPath); - - if (nodeModulesExist && !flatApplication.isSdkLayerStale) { + if ( + (await pathExists(sdkReadySentinelPath)) && + !flatApplication.isSdkLayerStale + ) { return; } - await fs.rm(sdkLayerPath, { recursive: true, force: true }); + const lockKey = `local-driver-sdk-layer:${flatApplication.workspaceId}:${applicationUniversalIdentifier}`; - const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk'); + await this.cacheLockService.withLock( + async () => { + const { flatApplicationMaps } = + await this.workspaceCacheService.getOrRecompute( + flatApplication.workspaceId, + ['flatApplicationMaps'], + ); + const freshFlatApplication = + flatApplicationMaps.byId[flatApplication.id]; + const isStale = freshFlatApplication?.isSdkLayerStale ?? true; - await this.sdkClientArchiveService.downloadAndExtractToPackage({ - workspaceId: flatApplication.workspaceId, - applicationId: flatApplication.id, - applicationUniversalIdentifier, - targetPackagePath: sdkPackagePath, - }); + if ((await pathExists(sdkReadySentinelPath)) && !isStale) { + return; + } - await this.sdkClientArchiveService.markSdkLayerFresh({ - applicationId: flatApplication.id, - workspaceId: flatApplication.workspaceId, - }); + await fs.rm(sdkLayerPath, { recursive: true, force: true }); + + const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk'); + + await this.sdkClientArchiveService.downloadAndExtractToPackage({ + workspaceId: flatApplication.workspaceId, + applicationId: flatApplication.id, + applicationUniversalIdentifier, + targetPackagePath: sdkPackagePath, + }); + + await this.sdkClientArchiveService.markSdkLayerFresh({ + applicationId: flatApplication.id, + workspaceId: flatApplication.workspaceId, + }); + + await fs.writeFile(sdkReadySentinelPath, ''); + }, + lockKey, + { + ttl: LAYER_BUILD_LOCK_TTL_MS, + ms: LAYER_BUILD_LOCK_RETRY_MS, + maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES, + }, + ); } async transpile({ diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts index 7694e7136d..53f7cd0a02 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory.ts @@ -17,6 +17,7 @@ import { DriverFactoryBase } from 'src/engine/core-modules/twenty-config/dynamic import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { ConfigGroupHashService } from 'src/engine/core-modules/twenty-config/services/config-group-hash.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; @Injectable() export class LogicFunctionDriverFactory extends DriverFactoryBase { @@ -26,6 +27,7 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase