houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6e319283c4749daaba74eeea2b6b8b14ca4d453f
twenty/packages/twenty-server/project.json
T
Charles Bochet 97871131a1 fix(server): mitigate integration-test OOM flakiness (#21588) 
## Problem

`server-integration-test` shards have been failing intermittently across
unrelated PRs with a distinctive signature: the shard exits code 1 with
**no jest assertion failure, no `Test Suites:` summary, and no V8
`JavaScript heap out of memory` error** — the process just dies mid-run.
Failures hit random shards and clear on re-run (e.g. an unrelated branch
failed shard 6 once, then passed 3× on identical code), while the
`merge_group` gate stays green.

### Root cause

Each shard runs a **single in-band jest process** that boots one shared
NestJS app (`globalSetup` → `app.listen`) and holds it for the entire
shard, driving heavy metadata migrations + cache rebuilds in that one
process. `NODE_OPTIONS=--max-old-space-size=12288` let V8 grow to 12 GB
— *above* the `ubuntu-latest` runner's available RAM (16 GB, shared with
Postgres/Redis/ClickHouse). V8 therefore deferred aggressive GC and grew
past physical memory, so the **OS OOM-killer killed the process before
V8 hit its own ceiling** — which is why there's no heap error and no
jest summary, just a silent exit.

## Changes (CI/test-only — prod runtime untouched)

- **Lower the integration jest heap cap `12288` → `6144`** so V8
self-limits below physical headroom instead of being OS-killed.
Counterintuitively safer: a real leak now surfaces as a *visible* heap
error naming the test, rather than a silent death. (`database:reset`
keeps 12288 — it runs alone, before jest.)
- **Add `--logHeapUsage`** to the integration jest runs to expose the
per-file heap trend for confirming/pinpointing the growth.
- **Split integration tests across 16 shards (was 10)** to lower the
peak working set per shard.
- **Make perf logging a first-class `LoggerService` tool** (per
@prastoin's review): add `LoggerService.perf()` and unify the existing
`time()`/`timeEnd()` helpers into `perfTime()`/`perfTimeEnd()` (now
routed through the driver), all gated by a new `PERF_LOG_ENABLED` config
var. It **defaults on** so real environments keep emitting the
install-perf logs, and `.env.test` sets it `false` to mute the
per-action flood in integration tests. The `application-manifest` and
`validate-build` services were moved from the built-in `Logger` to
`LoggerService` to use it.

## Notes

- `--max-old-space-size` lives only in the `test:integration` nx target;
it is **not** the prod server heap setting, so prod is unaffected.
- This is mitigation. If `--logHeapUsage` shows monotonic growth across
files, there's a real accumulation in the long-lived app (retained
flat-maps / metadata cache) worth a follow-up heap-snapshot fix.

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/21588?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->
2026-06-15 15:54:51 +02:00

302 lines
9.5 KiB
JSON
Raw Blame History

{
"name": "twenty-server",
"$schema": "../../node_modules/nx/schemas/project-schema.json",
"projectType": "application",
"tags": ["scope:backend"],
"targets": {
"build": {
"executor": "nx:run-commands",
"cache": true,
"options": {
"cwd": "packages/twenty-server",
"parallel": false,
"commands": [
"rimraf dist",
"nest build --path ./tsconfig.build.json",
"mkdir -p dist/assets/twenty-client-sdk && cp ../twenty-client-sdk/package.json dist/assets/twenty-client-sdk/ && cp -r ../twenty-client-sdk/dist dist/assets/twenty-client-sdk/dist"
]
},
"dependsOn": ["^build"]
},
"test:integration": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"commands": [
"NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=6144\" nx jest --config ./jest-integration.config.ts --logHeapUsage"
]
},
"parallel": false,
"configurations": {
"with-db-reset": {
"cwd": "packages/twenty-server",
"commands": [
"NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=12288 --import tsx/esm\" nx database:reset > reset-logs.log && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=6144\" nx jest --config ./jest-integration.config.ts --logHeapUsage"
]
}
}
},
"build:packageJson": {
"executor": "@nx/js:tsc",
"options": {
"main": "packages/twenty-server/dist/main.js",
"tsConfig": "packages/twenty-server/tsconfig.json",
"outputPath": "packages/twenty-server/dist",
"updateBuildableProjectDepsInPackageJson": true
}
},
"typecheck": {
"dependsOn": ["^build"]
},
"start": {
"executor": "nx:run-commands",
"cache": false,
"dependsOn": ["^build"],
"options": {
"cwd": "packages/twenty-server",
"command": "rimraf dist && NODE_ENV=development nest start --watch"
}
},
"start:ci": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "NODE_ENV=development nest start"
}
},
"start:ci-if-needed": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "packages/twenty-server",
"command": "curl -f -s http://localhost:3000/healthz > /dev/null 2>&1 && echo '✅ Server already running' || (echo '🚀 Server not running, starting...' && nohup nest start &)"
}
},
"start:debug": {
"executor": "nx:run-commands",
"dependsOn": ["^build"],
"options": {
"cwd": "packages/twenty-server",
"command": "rimraf dist && NODE_ENV=development nest start --watch --debug"
}
},
"reset:env": {
"executor": "nx:run-commands",
"inputs": ["{projectRoot}/.env.example"],
"outputs": ["{projectRoot}/.env"],
"cache": true,
"options": {
"cwd": "{projectRoot}",
"command": "cp .env.example .env"
}
},
"reset:env:e2e-testing-server": {
"executor": "nx:run-commands",
"inputs": ["{projectRoot}/.env.e2e-testing-server"],
"outputs": ["{projectRoot}/.env"],
"cache": true,
"options": {
"cwd": "{projectRoot}",
"command": "cp .env.e2e-testing-server .env"
}
},
"command": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "packages/twenty-server",
"command": "node dist/command/command.js"
}
},
"command-no-deps": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "node dist/command/command.js"
}
},
"worker": {
"executor": "nx:run-commands",
"dependsOn": ["^build"],
"options": {
"cwd": "packages/twenty-server",
"command": "NODE_ENV=development nest start --watch --entryFile queue-worker/queue-worker"
}
},
"ts-node": {
"executor": "nx:run-commands",
"dependsOn": ["^build"],
"options": {
"cwd": "packages/twenty-server",
"command": "ts-node"
}
},
"ts-node-no-deps": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "ts-node"
}
},
"ts-node-no-deps-transpile-only": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "ts-node --transpile-only"
}
},
"lint": {
"executor": "nx:run-commands",
"cache": true,
"dependsOn": ["twenty-oxlint-rules:build"],
"inputs": [
"{projectRoot}/src/**/*.{ts,tsx}",
"{projectRoot}/tsconfig*.json",
"{workspaceRoot}/tsconfig.base.json",
"{projectRoot}/.oxlintrc.json",
"{workspaceRoot}/packages/twenty-oxlint-rules/dist/oxlint-plugin.mjs"
],
"options": {
"cwd": "{projectRoot}",
"command": "npx oxlint --type-aware -c .oxlintrc.json src/ && (npx oxfmt --check src/ || (echo 'ERROR: oxfmt formatting check failed! Fix with: npx nx lint twenty-server --configuration=fix' && false))"
},
"configurations": {
"ci": {},
"fix": {
"command": "npx oxlint --type-aware --fix -c .oxlintrc.json src/ && npx oxfmt src/"
}
}
},
"lint:diff-with-main": {
"executor": "nx:run-commands",
"dependsOn": ["twenty-oxlint-rules:build"],
"options": {
"cwd": "{projectRoot}",
"command": "FILES=$(git diff --name-only --relative --diff-filter=d main...HEAD -- src/ | grep -E '\\.(ts|tsx)$'); [ -z \"$FILES\" ] && echo 'No changed files.' || (npx oxlint --type-aware -c .oxlintrc.json $FILES && (npx oxfmt --check $FILES || (echo 'ERROR: oxfmt formatting check failed! Fix with: npx nx lint:diff-with-main twenty-server --configuration=fix' && false)))"
},
"configurations": {
"fix": {
"command": "FILES=$(git diff --name-only --relative --diff-filter=d main...HEAD -- src/ | grep -E '\\.(ts|tsx)$'); [ -z \"$FILES\" ] && echo 'No changed files.' || (npx oxlint --type-aware --fix -c .oxlintrc.json $FILES && npx oxfmt $FILES)"
}
}
},
"test": {},
"test:debug": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register ../../node_modules/.bin/jest --runInBand"
}
},
"jest": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "jest"
}
},
"database:migrate": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "packages/twenty-server",
"command": "node dist/command/command.js run-instance-commands --force"
}
},
"database:init": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "packages/twenty-server",
"commands": [
"node dist/database/scripts/setup-db.js",
"nx database:migrate -- --include-slow"
],
"parallel": false
}
},
"database:migrate:generate": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "packages/twenty-server",
"command": "node dist/command/command.js generate:instance-command"
},
"cache": false
},
"generate:integration-test": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"commands": [
"nx ts-node-no-deps -- ./test/integration/graphql/codegen/index.ts"
],
"parallel": false
}
},
"database:reset": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"configurations": {
"no-seed": {
"cwd": "packages/twenty-server",
"commands": [
"node dist/database/scripts/truncate-db.js",
"nx database:init",
"nx command-no-deps -- cache:flush"
],
"parallel": false
},
"seed": {
"cwd": "packages/twenty-server",
"commands": [
"node dist/database/scripts/truncate-db.js",
"nx database:init",
"nx command-no-deps -- cache:flush",
"nx command-no-deps -- workspace:seed:dev"
],
"parallel": false
}
},
"defaultConfiguration": "seed"
},
"clickhouse:migrate": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "nx ts-node-no-deps-transpile-only -- src/database/clickHouse/migrations/run-migrations.ts"
}
},
"clickhouse:seed": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "nx ts-node-no-deps-transpile-only -- src/database/clickHouse/seeds/run-seeds.ts"
}
},
"lingui:extract": {
"executor": "nx:run-commands",
"dependsOn": ["^build"],
"options": {
"cwd": "{projectRoot}",
"command": "lingui extract --overwrite --clean"
}
},
"lingui:compile": {
"executor": "nx:run-commands",
"dependsOn": ["^build"],
"options": {
"cwd": "{projectRoot}",
"command": "lingui compile --typescript"
}
},
"version:bump": {
"executor": "nx:run-commands",
"options": {
"cwd": "{projectRoot}",
"command": "npx tsx scripts/bump-version.ts"
}
}
}
}
Reference in New Issue View Git Blame Copy Permalink