fix(logic-function): serialize LocalDriver layer builds with cache lock (#20054)
## Summary
Fixes a race condition in `LocalDriver` where concurrent `execute()`
calls for the same application can trash each other's layer build,
causing logic function executions to fail with either:
- `Error: ENOENT: process.cwd failed with error no such file or
directory, the current working directory was likely removed without
changing the working directory, uv_cwd` (the yarn-install child's `cwd`
got wiped mid-run), or
- `ENOENT: no such file or directory, open '…/<pkg>/<file>.js.map'`
raised from yarn's berry link step (files under the deps layer vanish
while yarn is extracting).
Both are thrown from `…/deps/<checksum>/.yarn/releases/yarn-4.9.2.cjs` —
i.e. the yarn process `copyYarnEngineAndBuildDependencies` spawns with
`cwd: buildDirectory`.
## Root cause
`LocalDriver.createLayerIfNotExist` (and `ensureSdkLayer`) both follow a
check-then-act pattern with no mutual exclusion:
```ts
if (await pathExists(depsNodeModulesPath)) return;
await fs.rm(depsLayerPath, { recursive: true, force: true });
await copyDependenciesInMemory(...);
await copyYarnEngineAndBuildDependencies(depsLayerPath); // spawns yarn with cwd=depsLayerPath
```
The deps layer path is shared across all `execute()` calls that match a
given `yarnLockChecksum`. Two concurrent callers (e.g. a webhook
invocation + a cron-triggered logic function firing while the first
run's layer is still being built) both see no `node_modules`, both
`fs.rm` the directory, and one's yarn child ends up with its `cwd` or
its extraction target gone.
## Fix
Wrap the critical sections with `CacheLockService.withLock` — the same
cache-backed lock the Lambda driver already uses for its layer/executor
builds:
- `createLayerIfNotExist` → lock key
`local-driver-deps-layer:${yarnLockChecksum ?? 'default'}`
- `ensureSdkLayer` → lock key
`local-driver-sdk-layer:${workspaceId}:${applicationUniversalIdentifier}`
A fast-path `pathExists` check is kept outside the lock (so warm
executions still skip lock acquisition entirely), and the existence +
staleness check is **repeated inside the lock** so followers no-op after
the leader finishes.
Lock parameters mirror the Lambda driver's layer-build lock (`ttl=120s`,
`retry=500ms`, `maxRetries=240`).
`cacheLockService` is now passed to `LocalDriver` from
`LogicFunctionDriverFactory` (it was already injected there for the
Lambda driver).
This commit is contained in:
+86
-27
@@ -14,6 +14,7 @@ import {
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
|
||||
|
||||
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
|
||||
import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
|
||||
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
|
||||
import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console';
|
||||
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
|
||||
@@ -22,10 +23,18 @@ import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-
|
||||
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
|
||||
import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
|
||||
import type { SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service';
|
||||
import type { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
|
||||
const LAYER_BUILD_LOCK_TTL_MS = 120_000;
|
||||
const LAYER_BUILD_LOCK_RETRY_MS = 500;
|
||||
const LAYER_BUILD_LOCK_MAX_RETRIES = 240;
|
||||
const LAYER_BUILD_READY_SENTINEL = '.twenty-layer-ready';
|
||||
|
||||
export interface LocalDriverOptions {
|
||||
logicFunctionResourceService: LogicFunctionResourceService;
|
||||
sdkClientArchiveService: SdkClientArchiveService;
|
||||
cacheLockService: CacheLockService;
|
||||
workspaceCacheService: WorkspaceCacheService;
|
||||
}
|
||||
|
||||
const pathExists = async (targetPath: string): Promise<boolean> => {
|
||||
@@ -41,10 +50,14 @@ const pathExists = async (targetPath: string): Promise<boolean> => {
|
||||
export class LocalDriver implements LogicFunctionDriver {
|
||||
private readonly logicFunctionResourceService: LogicFunctionResourceService;
|
||||
private readonly sdkClientArchiveService: SdkClientArchiveService;
|
||||
private readonly cacheLockService: CacheLockService;
|
||||
private readonly workspaceCacheService: WorkspaceCacheService;
|
||||
|
||||
constructor(options: LocalDriverOptions) {
|
||||
this.logicFunctionResourceService = options.logicFunctionResourceService;
|
||||
this.sdkClientArchiveService = options.sdkClientArchiveService;
|
||||
this.cacheLockService = options.cacheLockService;
|
||||
this.workspaceCacheService = options.workspaceCacheService;
|
||||
}
|
||||
|
||||
private getDepsLayerPath(flatApplication: FlatApplication): string {
|
||||
@@ -75,23 +88,40 @@ export class LocalDriver implements LogicFunctionDriver {
|
||||
applicationUniversalIdentifier: string;
|
||||
}): Promise<void> {
|
||||
const depsLayerPath = this.getDepsLayerPath(flatApplication);
|
||||
const depsNodeModulesPath = join(depsLayerPath, 'node_modules');
|
||||
const depsReadySentinelPath = join(
|
||||
depsLayerPath,
|
||||
LAYER_BUILD_READY_SENTINEL,
|
||||
);
|
||||
|
||||
const nodeModulesExist = await pathExists(depsNodeModulesPath);
|
||||
|
||||
if (nodeModulesExist) {
|
||||
if (await pathExists(depsReadySentinelPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Wipe any partial leftovers from a previously failed build
|
||||
await fs.rm(depsLayerPath, { recursive: true, force: true });
|
||||
const lockKey = `local-driver-deps-layer:${flatApplication.yarnLockChecksum ?? 'default'}`;
|
||||
|
||||
await this.logicFunctionResourceService.copyDependenciesInMemory({
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
inMemoryFolderPath: depsLayerPath,
|
||||
});
|
||||
await copyYarnEngineAndBuildDependencies(depsLayerPath);
|
||||
await this.cacheLockService.withLock(
|
||||
async () => {
|
||||
if (await pathExists(depsReadySentinelPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.rm(depsLayerPath, { recursive: true, force: true });
|
||||
|
||||
await this.logicFunctionResourceService.copyDependenciesInMemory({
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
inMemoryFolderPath: depsLayerPath,
|
||||
});
|
||||
await copyYarnEngineAndBuildDependencies(depsLayerPath);
|
||||
await fs.writeFile(depsReadySentinelPath, '');
|
||||
},
|
||||
lockKey,
|
||||
{
|
||||
ttl: LAYER_BUILD_LOCK_TTL_MS,
|
||||
ms: LAYER_BUILD_LOCK_RETRY_MS,
|
||||
maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async ensureSdkLayer({
|
||||
@@ -106,28 +136,57 @@ export class LocalDriver implements LogicFunctionDriver {
|
||||
applicationUniversalIdentifier,
|
||||
});
|
||||
const sdkNodeModulesPath = join(sdkLayerPath, 'node_modules');
|
||||
const sdkReadySentinelPath = join(sdkLayerPath, LAYER_BUILD_READY_SENTINEL);
|
||||
|
||||
const nodeModulesExist = await pathExists(sdkNodeModulesPath);
|
||||
|
||||
if (nodeModulesExist && !flatApplication.isSdkLayerStale) {
|
||||
if (
|
||||
(await pathExists(sdkReadySentinelPath)) &&
|
||||
!flatApplication.isSdkLayerStale
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.rm(sdkLayerPath, { recursive: true, force: true });
|
||||
const lockKey = `local-driver-sdk-layer:${flatApplication.workspaceId}:${applicationUniversalIdentifier}`;
|
||||
|
||||
const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk');
|
||||
await this.cacheLockService.withLock(
|
||||
async () => {
|
||||
const { flatApplicationMaps } =
|
||||
await this.workspaceCacheService.getOrRecompute(
|
||||
flatApplication.workspaceId,
|
||||
['flatApplicationMaps'],
|
||||
);
|
||||
const freshFlatApplication =
|
||||
flatApplicationMaps.byId[flatApplication.id];
|
||||
const isStale = freshFlatApplication?.isSdkLayerStale ?? true;
|
||||
|
||||
await this.sdkClientArchiveService.downloadAndExtractToPackage({
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
applicationId: flatApplication.id,
|
||||
applicationUniversalIdentifier,
|
||||
targetPackagePath: sdkPackagePath,
|
||||
});
|
||||
if ((await pathExists(sdkReadySentinelPath)) && !isStale) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.sdkClientArchiveService.markSdkLayerFresh({
|
||||
applicationId: flatApplication.id,
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
});
|
||||
await fs.rm(sdkLayerPath, { recursive: true, force: true });
|
||||
|
||||
const sdkPackagePath = join(sdkNodeModulesPath, 'twenty-client-sdk');
|
||||
|
||||
await this.sdkClientArchiveService.downloadAndExtractToPackage({
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
applicationId: flatApplication.id,
|
||||
applicationUniversalIdentifier,
|
||||
targetPackagePath: sdkPackagePath,
|
||||
});
|
||||
|
||||
await this.sdkClientArchiveService.markSdkLayerFresh({
|
||||
applicationId: flatApplication.id,
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
});
|
||||
|
||||
await fs.writeFile(sdkReadySentinelPath, '');
|
||||
},
|
||||
lockKey,
|
||||
{
|
||||
ttl: LAYER_BUILD_LOCK_TTL_MS,
|
||||
ms: LAYER_BUILD_LOCK_RETRY_MS,
|
||||
maxRetries: LAYER_BUILD_LOCK_MAX_RETRIES,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async transpile({
|
||||
|
||||
+4
@@ -17,6 +17,7 @@ import { DriverFactoryBase } from 'src/engine/core-modules/twenty-config/dynamic
|
||||
import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum';
|
||||
import { ConfigGroupHashService } from 'src/engine/core-modules/twenty-config/services/config-group-hash.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
|
||||
@Injectable()
|
||||
export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionDriver> {
|
||||
@@ -26,6 +27,7 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionD
|
||||
private readonly logicFunctionResourceService: LogicFunctionResourceService,
|
||||
private readonly sdkClientArchiveService: SdkClientArchiveService,
|
||||
private readonly cacheLockService: CacheLockService,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {
|
||||
super(twentyConfigService, configGroupHashService);
|
||||
}
|
||||
@@ -51,6 +53,8 @@ export class LogicFunctionDriverFactory extends DriverFactoryBase<LogicFunctionD
|
||||
return new LocalDriver({
|
||||
logicFunctionResourceService: this.logicFunctionResourceService,
|
||||
sdkClientArchiveService: this.sdkClientArchiveService,
|
||||
cacheLockService: this.cacheLockService,
|
||||
workspaceCacheService: this.workspaceCacheService,
|
||||
});
|
||||
|
||||
case LogicFunctionDriverType.LAMBDA: {
|
||||
|
||||
@@ -7,6 +7,7 @@ import { LogicFunctionTriggerModule } from 'src/engine/core-modules/logic-functi
|
||||
import { LogicFunctionExecutorModule } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module';
|
||||
import { SdkClientModule } from 'src/engine/core-modules/sdk-client/sdk-client.module';
|
||||
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
|
||||
@Global()
|
||||
@Module({})
|
||||
@@ -21,6 +22,7 @@ export class LogicFunctionModule {
|
||||
LogicFunctionTriggerModule,
|
||||
LogicFunctionExecutorModule,
|
||||
SdkClientModule,
|
||||
WorkspaceCacheModule,
|
||||
],
|
||||
providers: [LogicFunctionDriverFactory],
|
||||
exports: [
|
||||
|
||||
Reference in New Issue
Block a user