fix(logic-function): serialize LocalDriver layer builds with cache lock (#20054)

## Summary

Fixes a race condition in `LocalDriver` where concurrent `execute()`
calls for the same application can trash each other's layer build,
causing logic function executions to fail with either:

- `Error: ENOENT: process.cwd failed with error no such file or
directory, the current working directory was likely removed without
changing the working directory, uv_cwd` (the yarn-install child's `cwd`
got wiped mid-run), or
- `ENOENT: no such file or directory, open '…/<pkg>/<file>.js.map'`
raised from yarn's berry link step (files under the deps layer vanish
while yarn is extracting).

Both are thrown from `…/deps/<checksum>/.yarn/releases/yarn-4.9.2.cjs` —
i.e. the yarn process `copyYarnEngineAndBuildDependencies` spawns with
`cwd: buildDirectory`.

## Root cause

`LocalDriver.createLayerIfNotExist` (and `ensureSdkLayer`) both follow a
check-then-act pattern with no mutual exclusion:

```ts
if (await pathExists(depsNodeModulesPath)) return;
await fs.rm(depsLayerPath, { recursive: true, force: true });
await copyDependenciesInMemory(...);
await copyYarnEngineAndBuildDependencies(depsLayerPath); // spawns yarn with cwd=depsLayerPath
```

The deps layer path is shared across all `execute()` calls that match a
given `yarnLockChecksum`. Two concurrent callers (e.g. a webhook
invocation + a cron-triggered logic function firing while the first
run's layer is still being built) both see no `node_modules`, both
`fs.rm` the directory, and one's yarn child ends up with its `cwd` or
its extraction target gone.

## Fix

Wrap the critical sections with `CacheLockService.withLock` — the same
cache-backed lock the Lambda driver already uses for its layer/executor
builds:

- `createLayerIfNotExist` → lock key
`local-driver-deps-layer:${yarnLockChecksum ?? 'default'}`
- `ensureSdkLayer` → lock key
`local-driver-sdk-layer:${workspaceId}:${applicationUniversalIdentifier}`

A fast-path `pathExists` check is kept outside the lock (so warm
executions still skip lock acquisition entirely), and the existence +
staleness check is **repeated inside the lock** so followers no-op after
the leader finishes.

Lock parameters mirror the Lambda driver's layer-build lock (`ttl=120s`,
`retry=500ms`, `maxRetries=240`).

`cacheLockService` is now passed to `LocalDriver` from
`LogicFunctionDriverFactory` (it was already injected there for the
Lambda driver).
This commit is contained in:
Charles Bochet
2026-04-26 14:44:40 +02:00
committed by GitHub
parent ca84d28157
commit 499067ae14
3 changed files with 92 additions and 27 deletions
@@ -14,6 +14,7 @@ import {
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console';
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
@@ -22,10 +23,18 @@ import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
import type { SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service';
import type { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
const LAYER_BUILD_LOCK_TTL_MS = 120_000;
const LAYER_BUILD_LOCK_RETRY_MS = 500;
const LAYER_BUILD_LOCK_MAX_RETRIES = 240;
const LAYER_BUILD_READY_SENTINEL = '.twenty-layer-ready';
export interface LocalDriverOptions {
logicFunctionResourceService: LogicFunctionResourceService;
sdkClientArchiveService: SdkClientArchiveService;
cacheLockService: CacheLockService;
workspaceCacheService: WorkspaceCacheService;
}
const pathExists = async (targetPath: string): Promise<boolean> => {
@@ -41,10 +50,14 @@ const pathExists = async (targetPath: string): Promise<boolean> => {
export class LocalDriver implements LogicFunctionDriver {
private readonly logicFunctionResourceService: LogicFunctionResourceService;
private readonly sdkClientArchiveService: SdkClientArchiveService;
private readonly cacheLockService: CacheLockService;
private readonly workspaceCacheService: WorkspaceCacheService;
constructor(options: LocalDriverOptions) {
this.logicFunctionResourceService = options.logicFunctionResourceService;
this.sdkClientArchiveService = options.sdkClientArchiveService;
this.cacheLockService = options.cacheLockService;
this.workspaceCacheService = options.workspaceCacheService;
}
private getDepsLayerPath(flatApplication: FlatApplication): string {
@@ -75,23 +88,40 @@ export class LocalDriver implements LogicFunctionDriver {
applicationUniversalIdentifier: string;
}): Promise<void> {
const depsLayerPath = this.getDepsLayerPath(flatApplication);
const depsNodeModulesPath = join(depsLayerPath, 'node_modules');
const depsReadySentinelPath = join(
depsLayerPath,
LAYER_BUILD_READY_SENTINEL,
);
const nodeModulesExist = await pathExists(depsNodeModulesPath);
if (nodeModulesExist) {
if (await pathExists(depsReadySentinelPath)) {
return;
}
// Wipe any partial leftovers from a previously failed build
await fs.rm(depsLayerPath, { recursive: true, force: true });
const lockKey = `local-driver-deps-layer:${flatApplication.yarnLockChecksum ?? 'default'}`;
await this.logicFunctionResourceService.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatApplication.workspaceId,
inMemoryFolderPath: depsLayerPath,
});
await copyYarnEngineAndBuildDependencies(depsLayerPath);
await this.cacheLockService.withLock(
async () => {
if (await pathExists(depsReadySentinelPath)) {
return;
}
await fs.rm(depsLayerPath, { recursive: true, force: true });
await this.logicFunctionResourceService.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatApplication.workspaceId,
inMemoryFolderPath: depsLayerPath,
});
await copyYarnEngineAndBuildDependencies(depsLayerPath);
await fs.writeFile(depsReadySentinelPath, '');
},
lockKey,
{
ttl: LAYER_BUILD_LOCK_TTL_MS,
ms: LAYER_BUILD_LOCK_RETRY_MS,
maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES,
},
);
}
private async ensureSdkLayer({
@@ -106,28 +136,57 @@ export class LocalDriver implements LogicFunctionDriver {
applicationUniversalIdentifier,
});
const sdkNodeModulesPath = join(sdkLayerPath, 'node_modules');
const sdkReadySentinelPath = join(sdkLayerPath, LAYER_BUILD_READY_SENTINEL);
const nodeModulesExist = await pathExists(sdkNodeModulesPath);
if (nodeModulesExist && !flatApplication.isSdkLayerStale) {
if (
(await pathExists(sdkReadySentinelPath)) &&
!flatApplication.isSdkLayerStale
) {
return;
}
await fs.rm(sdkLayerPath, { recursive: true, force: true });
const lockKey = `local-driver-sdk-layer:${flatApplication.workspaceId}:${applicationUniversalIdentifier}`;
const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk');
await this.cacheLockService.withLock(
async () => {
const { flatApplicationMaps } =
await this.workspaceCacheService.getOrRecompute(
flatApplication.workspaceId,
['flatApplicationMaps'],
);
const freshFlatApplication =
flatApplicationMaps.byId[flatApplication.id];
const isStale = freshFlatApplication?.isSdkLayerStale ?? true;
await this.sdkClientArchiveService.downloadAndExtractToPackage({
workspaceId: flatApplication.workspaceId,
applicationId: flatApplication.id,
applicationUniversalIdentifier,
targetPackagePath: sdkPackagePath,
});
if ((await pathExists(sdkReadySentinelPath)) && !isStale) {
return;
}
await this.sdkClientArchiveService.markSdkLayerFresh({
applicationId: flatApplication.id,
workspaceId: flatApplication.workspaceId,
});
await fs.rm(sdkLayerPath, { recursive: true, force: true });
const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk');
await this.sdkClientArchiveService.downloadAndExtractToPackage({
workspaceId: flatApplication.workspaceId,
applicationId: flatApplication.id,
applicationUniversalIdentifier,
targetPackagePath: sdkPackagePath,
});
await this.sdkClientArchiveService.markSdkLayerFresh({
applicationId: flatApplication.id,
workspaceId: flatApplication.workspaceId,
});
await fs.writeFile(sdkReadySentinelPath, '');
},
lockKey,
{
ttl: LAYER_BUILD_LOCK_TTL_MS,
ms: LAYER_BUILD_LOCK_RETRY_MS,
maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES,
},
);
}
async transpile({
@@ -17,6 +17,7 @@ import { DriverFactoryBase } from 'src/engine/core-modules/twenty-config/dynamic
import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum';
import { ConfigGroupHashService } from 'src/engine/core-modules/twenty-config/services/config-group-hash.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
@Injectable()
export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionDriver> {
@@ -26,6 +27,7 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionD
private readonly logicFunctionResourceService: LogicFunctionResourceService,
private readonly sdkClientArchiveService: SdkClientArchiveService,
private readonly cacheLockService: CacheLockService,
private readonly workspaceCacheService: WorkspaceCacheService,
) {
super(twentyConfigService, configGroupHashService);
}
@@ -51,6 +53,8 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionD
return new LocalDriver({
logicFunctionResourceService: this.logicFunctionResourceService,
sdkClientArchiveService: this.sdkClientArchiveService,
cacheLockService: this.cacheLockService,
workspaceCacheService: this.workspaceCacheService,
});
case LogicFunctionDriverType.LAMBDA: {
@@ -7,6 +7,7 @@ import { LogicFunctionTriggerModule } from 'src/engine/core-modules/logic-functi
import { LogicFunctionExecutorModule } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module';
import { SdkClientModule } from 'src/engine/core-modules/sdk-client/sdk-client.module';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@Global()
@Module({})
@@ -21,6 +22,7 @@ export class LogicFunctionModule {
LogicFunctionTriggerModule,
LogicFunctionExecutorModule,
SdkClientModule,
WorkspaceCacheModule,
],
providers: [LogicFunctionDriverFactory],
exports: [