houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix sdk-e2e-test: ensure DB is ready before server starts (#19583)

Browse Source 
## Summary

- The `sdk-e2e-test` CI job has been failing since at least April 9th
because the nx dependency graph runs `database:reset` and
`start:ci-if-needed` in **parallel** — neither depends on the other. The
server starts before the DB tables are created, crashes with `relation
"core.keyValuePair" does not exist`, and `wait-on` times out after 10
minutes.
- Replace the `nx-affected` orchestration for e2e with explicit
**sequential** CI steps: build → create DB → reset DB → start server →
wait for health → run tests.
- Add server log dump on failure for easier future debugging.

## Root cause

In `packages/twenty-sdk/project.json`, the `test:e2e` target has:

```json
"dependsOn": [
  "build",
  { "target": "database:reset", "projects": "twenty-server" },
  { "target": "start:ci-if-needed", "projects": "twenty-server" }
]
```

Since `database:reset` and `start:ci-if-needed` don't depend on each
other, nx can (and does) run them concurrently. `start:ci-if-needed`
fires `nohup nest start &` and immediately completes. The server process
tries to query `core.keyValuePair` before `database:reset` creates it →
crash → wait-on timeout → job failure.
This commit is contained in:
Charles Bochet
2026-04-11 16:02:12 +02:00
committed by GitHub
parent 3b55026452
commit baf2fc4cc9
14 changed files with 290 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-sdk.yaml
+17 -7
View File
@@ -19,6 +19,7 @@ jobs:
files: |
packages/twenty-sdk/**
packages/twenty-server/**
.github/workflows/ci-sdk.yaml
!packages/twenty-sdk/package.json
sdk-test:
needs: changed-files-check
@@ -69,7 +70,6 @@ jobs:
ports:
- 6379:6379
env:
NODE_ENV: test
TWENTY_API_URL: http://localhost:3000
TWENTY_API_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIyMDIwMjAyMC1lNmI1LTQ2ODAtOGEzMi1iODIwOTczNzE1NmIiLCJ1c2VySWQiOiIyMDIwMjAyMC1lNmI1LTQ2ODAtOGEzMi1iODIwOTczNzE1NmIiLCJ3b3Jrc3BhY2VJZCI6IjIwMjAyMDIwLTFjMjUtNGQwMi1iZjI1LTZhZWNjZjdlYTQxOSIsIndvcmtzcGFjZU1lbWJlcklkIjoiMjAyMDIwMjAtNDYzZi00MzViLTgyOGMtMTA3ZTAwN2EyNzExIiwidXNlcldvcmtzcGFjZUlkIjoiMjAyMDIwMjAtMWU3Yy00M2Q5LWE1ZGItNjg1YjUwNjlkODE2IiwidHlwZSI6IkFDQ0VTUyIsImF1dGhQcm92aWRlciI6InBhc3N3b3JkIiwiaWF0IjoxNzUxMjgxNzA0LCJleHAiOjIwNjY4NTc3MDR9.HMGqCsVlOAPVUBhKSGlD1X86VoHKt4LIUtET3CGIdik
steps:
@@ -79,16 +79,26 @@ jobs:
fetch-depth: 10
- name: Install dependencies
uses: ./.github/actions/yarn-install
- name: Build
- name: Build SDK
run: npx nx build twenty-sdk
- name: Server / Create Test DB
- name: Setup server environment
run: npx nx reset:env:e2e-testing-server twenty-server
- name: Create databases
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup database
run: npx nx run twenty-server:database:reset
- name: Start server
run: nohup npx nx start:ci twenty-server > /tmp/twenty-server.log 2>&1 &
- name: Wait for server to be ready
run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
- name: SDK / Run e2e Tests
uses: ./.github/actions/nx-affected
with:
tag: scope:sdk
tasks: test:e2e
run: NODE_ENV=test npx vitest run --config ./vitest.e2e.config.ts
working-directory: packages/twenty-sdk
- name: Server / Dump logs on failure
if: failure()
run: tail -100 /tmp/twenty-server.log || echo "No server log file found"
ci-sdk-status-check:
if: always() && !cancelled()
timeout-minutes: 5
packages/twenty-sdk/src/cli/__tests__/constants/setupTest.ts
-2
View File
@@ -43,6 +43,4 @@ beforeAll(async () => {
2,
),
);
process.env.TWENTY_APP_ACCESS_TOKEN ??= token;
});
packages/twenty-sdk/src/cli/__tests__/integration/utils/setup-app-dev-mocks.ts
+7 -2
View File
@@ -56,14 +56,19 @@ vi.mock('@/cli/utilities/file/file-uploader', () => ({
},
}));
vi.mock('@/cli/utilities/auth/resolve-app-access-token', () => ({
ensureValidAppAccessTokenOrRefresh: vi
vi.mock('@/cli/utilities/auth', () => ({
ensureAppAccessTokenIsValidOrRefresh: vi
.fn()
.mockResolvedValue('mock-app-access-token'),
exchangeCredentialsForTokens: vi.fn().mockResolvedValue({
accessToken: 'mock-app-access-token',
refreshToken: 'mock-app-refresh-token',
}),
ensureAppRegistration: vi.fn().mockResolvedValue({
clientId: 'mock-client-id',
clientSecret: 'mock-client-secret',
isNewRegistration: true,
}),
}));
vi.mock('@/cli/utilities/client/client-service', () => ({
packages/twenty-sdk/src/cli/operations/dev-once.ts
+16 -38
View File
@@ -2,15 +2,15 @@ import path from 'path';
import { OUTPUT_DIR, type Manifest } from 'twenty-shared/application';
import { ApiService } from '@/cli/utilities/api/api-service';
import {
ensureAppAccessTokenIsValidOrRefresh,
ensureAppRegistration,
} from '@/cli/utilities/auth';
import { buildApplication } from '@/cli/utilities/build/common/build-application';
import { runTypecheck } from '@/cli/utilities/build/common/typecheck-plugin';
import { buildAndValidateManifest } from '@/cli/utilities/build/manifest/build-and-validate-manifest';
import { manifestUpdateChecksums } from '@/cli/utilities/build/manifest/manifest-update-checksums';
import { writeManifestToOutput } from '@/cli/utilities/build/manifest/manifest-writer';
import {
ensureValidAppAccessTokenOrRefresh,
exchangeCredentialsForTokens,
} from '@/cli/utilities/auth/resolve-app-access-token';
import { ClientService } from '@/cli/utilities/client/client-service';
import { ConfigService } from '@/cli/utilities/config/config-service';
import { formatSyncErrorEvents } from '@/cli/utilities/dev/orchestrator/steps/format-sync-error-events';
@@ -124,37 +124,14 @@ const innerAppDevOnce = async (
const configService = new ConfigService();
const appAccessToken =
await ensureValidAppAccessTokenOrRefresh(configService);
if (!appAccessToken) {
const createResult = await apiService.createApplicationRegistration({
const { clientId, clientSecret } = await ensureAppRegistration(
apiService,
configService,
{
name: manifest.application.displayName,
universalIdentifier: manifest.application.universalIdentifier,
});
if (!createResult.success) {
return {
success: false,
error: {
code: APP_ERROR_CODES.SYNC_FAILED,
message: `Failed to create app registration: ${serializeError(createResult.error)}`,
},
};
}
const { applicationRegistration, clientSecret } = createResult.data;
await configService.setConfig({
appRegistrationId: applicationRegistration.id,
appRegistrationClientId: applicationRegistration.oAuthClientId,
});
await exchangeCredentialsForTokens(configService, {
clientId: applicationRegistration.oAuthClientId,
clientSecret,
});
}
},
);
const createDevAppResult = await apiService.createDevelopmentApplication({
universalIdentifier: manifest.application.universalIdentifier,
@@ -166,7 +143,7 @@ const innerAppDevOnce = async (
success: false,
error: {
code: APP_ERROR_CODES.SYNC_FAILED,
message: `Failed to create development application: ${serializeError(createDevAppResult.error)}`,
message: `Failed to install development application: ${serializeError(createDevAppResult.error)}`,
},
};
}
@@ -235,13 +212,14 @@ const innerAppDevOnce = async (
};
}
// Generate the CoreApiClient using an APPLICATION_ACCESS token so the
// server returns the app-scoped schema (objects defined by this app).
onProgress?.('Generating API client...');
try {
const appAccessToken =
await ensureValidAppAccessTokenOrRefresh(configService);
const appAccessToken = await ensureAppAccessTokenIsValidOrRefresh(
configService,
{ clientId, clientSecret },
);
const clientService = new ClientService();
await clientService.generateCoreClient({
packages/twenty-sdk/src/cli/utilities/auth/ensure-app-access-token-is-valid-or-refresh.ts
+65
View File
@@ -0,0 +1,65 @@
import { type ConfigService } from '@/cli/utilities/config/config-service';
import { exchangeCredentialsForTokens } from './exchange-credentials-for-tokens';
const EXPIRATION_MARGIN_MS = 30_000;
const isTokenExpired = (token: string): boolean => {
try {
const payload = JSON.parse(
Buffer.from(token.split('.')[1], 'base64').toString(),
);
return payload.exp * 1_000 < Date.now() + EXPIRATION_MARGIN_MS;
} catch {
return false;
}
};
export const ensureAppAccessTokenIsValidOrRefresh = async (
configService: ConfigService,
credentials?: { clientId: string; clientSecret: string },
): Promise<string | undefined> => {
const config = await configService.getConfig();
if (config.appAccessToken && !isTokenExpired(config.appAccessToken)) {
return config.appAccessToken;
}
if (config.appRefreshToken && config.appRegistrationClientId) {
const response = await fetch(`${config.apiUrl}/oauth/token`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
grant_type: 'refresh_token',
refresh_token: config.appRefreshToken,
client_id: config.appRegistrationClientId,
}),
});
if (response.ok) {
const data = (await response.json()) as {
access_token: string;
refresh_token?: string;
};
await configService.setConfig({
appAccessToken: data.access_token,
...(data.refresh_token ? { appRefreshToken: data.refresh_token } : {}),
});
return data.access_token;
}
}
if (credentials) {
const result = await exchangeCredentialsForTokens(
configService,
credentials,
);
return result.accessToken;
}
return undefined;
};
packages/twenty-sdk/src/cli/utilities/auth/ensure-app-registration.ts
+89
View File
@@ -0,0 +1,89 @@
import { type ApiService } from '@/cli/utilities/api/api-service';
import { type ConfigService } from '@/cli/utilities/config/config-service';
import { isDefined, isPlainObject } from 'twenty-shared/utils';
const isAlreadyClaimedError = (error: unknown): boolean => {
if (!isPlainObject(error)) {
return false;
}
const { extensions } = error as { extensions?: { subCode?: string } };
return (
isDefined(extensions) &&
extensions.subCode === 'UNIVERSAL_IDENTIFIER_ALREADY_CLAIMED'
);
};
export const ensureAppRegistration = async (
apiService: ApiService,
configService: ConfigService,
app: { name: string; universalIdentifier: string },
): Promise<{
clientId: string;
clientSecret: string;
isNewRegistration: boolean;
}> => {
const createResult = await apiService.createApplicationRegistration({
name: app.name,
universalIdentifier: app.universalIdentifier,
});
if (createResult.success) {
const { applicationRegistration, clientSecret } = createResult.data;
await configService.setConfig({
appRegistrationId: applicationRegistration.id,
appRegistrationClientId: applicationRegistration.oAuthClientId,
});
return {
clientId: applicationRegistration.oAuthClientId,
clientSecret,
isNewRegistration: true,
};
}
if (!isAlreadyClaimedError(createResult.error)) {
const errorDetail =
createResult.error instanceof Error
? createResult.error.message
: ((createResult.error as { message?: string })?.message ??
String(createResult.error));
throw new Error(`Failed to create app registration: ${errorDetail}`);
}
const findResult =
await apiService.findApplicationRegistrationByUniversalIdentifier(
app.universalIdentifier,
);
if (!findResult.success || !findResult.data) {
throw new Error(
`App registration exists but could not be found: ${app.universalIdentifier}`,
);
}
const registration = findResult.data;
await configService.setConfig({
appRegistrationId: registration.id,
appRegistrationClientId: registration.oAuthClientId,
});
const rotateResult =
await apiService.rotateApplicationRegistrationClientSecret(registration.id);
if (!rotateResult.success || !rotateResult.data) {
throw new Error(
`Failed to rotate client secret for registration ${registration.id}`,
);
}
return {
clientId: registration.oAuthClientId,
clientSecret: rotateResult.data.clientSecret,
isNewRegistration: false,
};
};
packages/twenty-sdk/src/cli/utilities/auth/exchange-credentials-for-tokens.ts
+39
View File
@@ -0,0 +1,39 @@
import { type ConfigService } from '@/cli/utilities/config/config-service';
export const exchangeCredentialsForTokens = async (
configService: ConfigService,
params: { clientId: string; clientSecret: string },
): Promise<{ accessToken: string; refreshToken?: string }> => {
const config = await configService.getConfig();
const response = await fetch(`${config.apiUrl}/oauth/token`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
grant_type: 'client_credentials',
client_id: params.clientId,
client_secret: params.clientSecret,
}),
});
if (!response.ok) {
throw new Error(
`Token exchange failed: ${response.status} ${response.statusText}`,
);
}
const data = (await response.json()) as {
access_token: string;
refresh_token?: string;
};
await configService.setConfig({
appAccessToken: data.access_token,
...(data.refresh_token ? { appRefreshToken: data.refresh_token } : {}),
});
return {
accessToken: data.access_token,
refreshToken: data.refresh_token,
};
};
packages/twenty-sdk/src/cli/utilities/auth/index.ts
+3
View File
@@ -0,0 +1,3 @@
export { ensureAppAccessTokenIsValidOrRefresh } from './ensure-app-access-token-is-valid-or-refresh';
export { ensureAppRegistration } from './ensure-app-registration';
export { exchangeCredentialsForTokens } from './exchange-credentials-for-tokens';
packages/twenty-sdk/src/cli/utilities/auth/resolve-app-access-token.ts
-104
View File
@@ -1,104 +0,0 @@
import { type ConfigService } from '@/cli/utilities/config/config-service';
const EXPIRATION_MARGIN_MS = 30_000;
const isTokenExpired = (token: string): boolean => {
try {
const payload = JSON.parse(
Buffer.from(token.split('.')[1], 'base64').toString(),
);
return payload.exp * 1_000 < Date.now() + EXPIRATION_MARGIN_MS;
} catch {
return false;
}
};
/**
* Exchanges an app registration's clientId + clientSecret for access/refresh
* tokens via the OAuth client_credentials grant, then persists them in config.
*/
export const exchangeCredentialsForTokens = async (
configService: ConfigService,
params: { clientId: string; clientSecret: string },
): Promise<{ accessToken: string; refreshToken?: string }> => {
const config = await configService.getConfig();
const response = await fetch(`${config.apiUrl}/oauth/token`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
grant_type: 'client_credentials',
client_id: params.clientId,
client_secret: params.clientSecret,
}),
});
if (!response.ok) {
throw new Error(
`Token exchange failed: ${response.status} ${response.statusText}`,
);
}
const data = (await response.json()) as {
access_token: string;
refresh_token?: string;
};
await configService.setConfig({
appAccessToken: data.access_token,
...(data.refresh_token ? { appRefreshToken: data.refresh_token } : {}),
});
return {
accessToken: data.access_token,
refreshToken: data.refresh_token,
};
};
/**
* Returns a valid appAccessToken from config, refreshing it first if expired.
*/
export const ensureValidAppAccessTokenOrRefresh = async (
configService: ConfigService,
): Promise<string | undefined> => {
const config = await configService.getConfig();
if (!config.appAccessToken) {
return undefined;
}
if (!isTokenExpired(config.appAccessToken)) {
return config.appAccessToken;
}
if (!config.appRefreshToken || !config.appRegistrationClientId) {
return undefined;
}
const response = await fetch(`${config.apiUrl}/oauth/token`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
grant_type: 'refresh_token',
refresh_token: config.appRefreshToken,
client_id: config.appRegistrationClientId,
}),
});
if (!response.ok) {
return undefined;
}
const data = (await response.json()) as {
access_token: string;
refresh_token?: string;
};
await configService.setConfig({
appAccessToken: data.access_token,
...(data.refresh_token ? { appRefreshToken: data.refresh_token } : {}),
});
return data.access_token;
};
packages/twenty-sdk/src/cli/utilities/dev/orchestrator/dev-mode-orchestrator.ts
+3 -2
View File
@@ -215,6 +215,7 @@ export class DevModeOrchestrator {
if (objectsOrFieldsChanged) {
await this.generateApiClientStep.execute({
appPath: this.state.appPath,
credentials: this.registerAppStep.registrationCredentials,
});
this.skipTypecheck = false;
@@ -232,7 +233,7 @@ export class DevModeOrchestrator {
if (!createResult.success || !createResult.data) {
this.state.applyStepEvents([
{
message: 'Failed to create development application',
message: 'Failed to install development application',
status: 'error',
},
{ message: JSON.stringify(createResult, null, 2), status: 'error' },
@@ -249,7 +250,7 @@ export class DevModeOrchestrator {
this.state.steps.resolveApplication.status = 'done';
this.state.applyStepEvents([
{ message: 'Application created', status: 'success' },
{ message: 'Application installed', status: 'success' },
]);
this.uploadFilesStep.initialize({
packages/twenty-sdk/src/cli/utilities/dev/orchestrator/steps/generate-api-client-orchestrator-step.ts
+7 -3
View File
@@ -1,4 +1,4 @@
import { ensureValidAppAccessTokenOrRefresh } from '@/cli/utilities/auth/resolve-app-access-token';
import { ensureAppAccessTokenIsValidOrRefresh } from '@/cli/utilities/auth';
import { type ClientService } from '@/cli/utilities/client/client-service';
import { type ConfigService } from '@/cli/utilities/config/config-service';
import { type OrchestratorState } from '@/cli/utilities/dev/orchestrator/dev-mode-orchestrator-state';
@@ -26,15 +26,19 @@ export class GenerateApiClientOrchestratorStep {
this.notify = notify;
}
async execute(input: { appPath: string }): Promise<void> {
async execute(input: {
appPath: string;
credentials?: { clientId: string; clientSecret: string };
}): Promise<void> {
const step = this.state.steps.generateApiClient;
step.status = 'in_progress';
this.notify();
try {
const appAccessToken = await ensureValidAppAccessTokenOrRefresh(
const appAccessToken = await ensureAppAccessTokenIsValidOrRefresh(
this.configService,
input.credentials,
);
await this.clientService.generateCoreClient({
packages/twenty-sdk/src/cli/utilities/dev/orchestrator/steps/register-app-orchestrator-step.ts
+41 -51
View File
@@ -1,8 +1,5 @@
import { type ApiService } from '@/cli/utilities/api/api-service';
import {
ensureValidAppAccessTokenOrRefresh,
exchangeCredentialsForTokens,
} from '@/cli/utilities/auth/resolve-app-access-token';
import { ensureAppRegistration } from '@/cli/utilities/auth';
import { type ConfigService } from '@/cli/utilities/config/config-service';
import { type OrchestratorState } from '@/cli/utilities/dev/orchestrator/dev-mode-orchestrator-state';
import { type Manifest } from 'twenty-shared/application';
@@ -13,6 +10,10 @@ export class RegisterAppOrchestratorStep {
private state: OrchestratorState;
private notify: () => void;
registrationCredentials:
| { clientId: string; clientSecret: string }
| undefined;
constructor({
apiService,
configService,
@@ -31,59 +32,48 @@ export class RegisterAppOrchestratorStep {
}
async execute(input: { manifest: Manifest }): Promise<void> {
const existingToken = await ensureValidAppAccessTokenOrRefresh(
this.configService,
);
try {
const reg = await ensureAppRegistration(
this.apiService,
this.configService,
{
name: input.manifest.application.displayName,
universalIdentifier: input.manifest.application.universalIdentifier,
},
);
this.registrationCredentials = {
clientId: reg.clientId,
clientSecret: reg.clientSecret,
};
if (existingToken) {
this.state.applyStepEvents([
{ message: 'App registration found in config', status: 'info' },
{
message: reg.isNewRegistration
? `App registration created: ${input.manifest.application.displayName}`
: 'Existing app registration found',
status: reg.isNewRegistration ? 'success' : 'info',
},
...(reg.isNewRegistration
? [
{
message: 'Credentials saved to config.' as const,
status: 'info' as const,
},
]
: []),
]);
} catch (error) {
this.state.applyStepEvents([
{
message: `Failed to register app: ${error instanceof Error ? error.message : String(error)}`,
status: 'error',
},
]);
this.notify();
return;
throw error;
}
const createResult = await this.apiService.createApplicationRegistration({
name: input.manifest.application.displayName,
universalIdentifier: input.manifest.application.universalIdentifier,
});
if (!createResult.success || !createResult.data) {
this.state.applyStepEvents([
{ message: 'Failed to create app registration', status: 'warning' },
]);
this.notify();
return;
}
const { applicationRegistration, clientSecret } = createResult.data;
await this.configService.setConfig({
appRegistrationId: applicationRegistration.id,
appRegistrationClientId: applicationRegistration.oAuthClientId,
});
await exchangeCredentialsForTokens(this.configService, {
clientId: applicationRegistration.oAuthClientId,
clientSecret,
});
this.state.applyStepEvents([
{
message: `App registration created: ${input.manifest.application.displayName}`,
status: 'success',
},
{
message: `Client ID: ${applicationRegistration.oAuthClientId}`,
status: 'info',
},
{
message: 'Credentials saved to config.',
status: 'info',
},
]);
this.notify();
}
}
packages/twenty-sdk/src/cli/utilities/dev/orchestrator/steps/upload-files-orchestrator-step.ts
+2 -1
View File
@@ -4,6 +4,7 @@ import {
} from '@/cli/utilities/dev/orchestrator/dev-mode-orchestrator-state';
import { FileUploader } from '@/cli/utilities/file/file-uploader';
import { type FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
export type UploadFilesOrchestratorStepOutput = {
fileUploader: FileUploader | null;
@@ -34,7 +35,7 @@ export class UploadFilesOrchestratorStep {
}
get isInitialized(): boolean {
return this.state.steps.uploadFiles.output.fileUploader !== null;
return isDefined(this.state.steps.uploadFiles.output.fileUploader);
}
initialize(input: { appPath: string; universalIdentifier: string }): void {
packages/twenty-sdk/vitest.e2e.config.ts
+1 -1
View File
@@ -25,7 +25,7 @@ export default defineConfig({
concurrent: false,
},
env: {
TWENTY_API_URL: process.env.TWENTY_API_URL ?? 'http://localhost:2021',
TWENTY_API_URL: process.env.TWENTY_API_URL ?? 'http://localhost:2020',
TWENTY_API_KEY:
process.env.TWENTY_API_KEY ??
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIyMDIwMjAyMC0xYzI1LTRkMDItYmYyNS02YWVjY2Y3ZWE0MTkiLCJ0eXBlIjoiQVBJX0tFWSIsIndvcmtzcGFjZUlkIjoiMjAyMDIwMjAtMWMyNS00ZDAyLWJmMjUtNmFlY2NmN2VhNDE5IiwiaWF0IjoxNzM1Njg5NjAwLCJleHAiOjQ4OTE0NDk2MDAsImp0aSI6IjIwMjAyMDIwLWY0MDEtNGQ4YS1hNzMxLTY0ZDAwN2MyN2JhZCJ9.bfQjfyN0NEtTCLE_xPyNcwonDzlSXFoP8kdCQTdnuDc',