From 439adc6ac0e376fbedbfc13e455221df62308a7e Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Mon, 24 Nov 2025 18:20:07 +0100 Subject: [PATCH] Restore transaction on `WorkspaceCustomApplicationIdNonNullable1763977334519` failure (#16032) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Introduction When a transaction query fails it gets aborted, even if we catch the js exception typeorm ack it and fails By adding a save point we isolate the issue and restore the transaction 🥷 ## Through database:migrate:prod Tested but lost logs ## Upgrade ```ts ➜ twenty-server git:(fix-migration-runner) ✗ npx nx command twenty-server upgrade ✔ 3/3 dependent project tasks succeeded [3 read from cache] Hint: you can run the command with --verbose to see the full dependent project outputs ——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————— > nx run twenty-server:build [existing outputs match the cache, left as is] > rimraf dist > nest build --path ./tsconfig.build.json > SWC Running... Successfully compiled: 3747 files with swc (65.82ms) > nx run twenty-server:command upgrade query: SELECT * FROM current_schema() query: SELECT version(); [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [NestFactory] Starting Nest application... [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [InstanceLoader] CommandRootModule dependencies initialized [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [InstanceLoader] CommandModule dependencies initialized // ... [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [InstanceLoader] WorkflowApiModule dependencies initialized [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [InstanceLoader] AuthModule dependencies initialized [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [PgPoolSharedService] Pool sharing will use max 10 connections per pool with 600000ms idle timeout and allowExitOnIdle=true [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [PgPoolSharedService] pg.Pool patched successfully by this service instance. [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [PgPoolSharedService] Pg pool sharing initialized - pools will be shared across tenants [Nest] 82844 - 11/24/2025, 6:05:11 PM DEBUG [PgPoolSharedService] No active pg pools to log stats for [Nest] 82844 - 11/24/2025, 6:05:11 PM DEBUG [PgPoolSharedService] Pool statistics logging enabled (30s interval) [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [PgPoolSharedService] Created new shared pg Pool for key "localhost|5432|postgres||no-ssl" with 10 max connections and 600000 ms idle timeout. Total pools: 1 [Nest] 82844 - 11/24/2025, 6:05:11 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: New connection established [Nest] 82844 - 11/24/2025, 6:05:11 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool [Nest] 82844 - 11/24/2025, 6:05:11 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool [Nest] 82844 - 11/24/2025, 6:05:11 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [DatabaseConfigDriver] [INIT] Loading initial config variables from database [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [DatabaseConfigDriver] [INIT] Config variables loaded: 0 values found in DB, 59 falling to env vars/defaults [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [UpgradeCommand] Initialized upgrade context with: - currentVersion (migrating to): 1.12.0 - fromWorkspaceVersion: 1.11.0 - 3 commands [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [UpgradeCommand] Running global database migrations [Nest] 82844 - 11/24/2025, 6:05:11 PM LOG [UpgradeCommand] Running core datasource migrations... [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [UpgradeCommand] query: SELECT * FROM current_schema() query: CREATE EXTENSION IF NOT EXISTS "uuid-ossp" query: SELECT version(); query: SELECT * FROM "information_schema"."tables" WHERE "table_schema" = 'core' AND "table_name" = '_typeorm_migrations' query: SELECT * FROM "core"."_typeorm_migrations" "_typeorm_migrations" ORDER BY "id" DESC 46 migrations are already loaded in the database. 47 migrations were found in the source code. AddCanBeUninstalledColumnToApplication1763731277403 is the last executed migration. It was executed on Fri Nov 21 2025 14:21:17 GMT+0100 (Central European Standard Time). 1 migrations are new migrations must be executed. query: START TRANSACTION query: SAVEPOINT sp_workspace_custom_application_id_non_nullable query: ALTER TABLE "core"."workspace" DROP CONSTRAINT "FK_3b1acb13a5dac9956d1a4b32755" query: ALTER TABLE "core"."workspace" ALTER COLUMN "workspaceCustomApplicationId" SET NOT NULL query failed: ALTER TABLE "core"."workspace" ALTER COLUMN "workspaceCustomApplicationId" SET NOT NULL error: error: column "workspaceCustomApplicationId" of relation "workspace" contains null values query: ROLLBACK TO SAVEPOINT sp_workspace_custom_application_id_non_nullable query: RELEASE SAVEPOINT sp_workspace_custom_application_id_non_nullable query: INSERT INTO "core"."_typeorm_migrations"("timestamp", "name") VALUES ($1, $2) -- PARAMETERS: [1763977334519,"WorkspaceCustomApplicationIdNonNullable1763977334519"] Migration WorkspaceCustomApplicationIdNonNullable1763977334519 has been executed successfully. query: COMMIT [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [UpgradeCommand] Database migrations completed successfully [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [UpgradeCommand] Running command on workspace 20202020-1c25-4d02-bf25-6aeccf7ea419 1/2 Computing new Datasource for cacheKey: 20202020-1c25-4d02-bf25-6aeccf7ea419-10 out of 0 [Nest] 82844 - 11/24/2025, 6:05:12 PM DEBUG [PgPoolSharedService] Reusing existing pg Pool for key "localhost|5432|postgres||no-ssl" [Nest] 82844 - 11/24/2025, 6:05:12 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool [Nest] 82844 - 11/24/2025, 6:05:12 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool query: SELECT * FROM current_schema() [Nest] 82844 - 11/24/2025, 6:05:12 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool query: SELECT version(); [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [UpgradeCommand] Upgrading workspace 20202020-1c25-4d02-bf25-6aeccf7ea419 from=1.11.0 to=1.12.0 1/2 [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [CreateWorkspaceCustomApplicationCommand] Checking standard applications for workspace 20202020-1c25-4d02-bf25-6aeccf7ea419 [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [CreateWorkspaceCustomApplicationCommand] 20202020-1c25-4d02-bf25-6aeccf7ea419 skipping custom workspace application creation as already exists query failed: ALTER TABLE "core"."workspace" ALTER COLUMN "workspaceCustomApplicationId" SET NOT NULL error: error: column "workspaceCustomApplicationId" of relation "workspace" contains null values [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceCustomApplicationIdNonNullableCommand] Rollbacking WorkspaceCustomApplicationIdNonNullableCommand: column "workspaceCustomApplicationId" of relation "workspace" contains null values [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [SyncWorkspaceMetadataCommand] Running workspace sync for workspace: 20202020-1c25-4d02-bf25-6aeccf7ea419 (0 out of 2) [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncMetadataService] Syncing standard objects and fields metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncMetadataService] Syncing standard objects and fields metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncObjectMetadataService] Comparing standard objects and fields metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncObjectMetadataService] Updating workspace metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncObjectMetadataService] Generating migrations [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncObjectMetadataService] Saving migrations [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncMetadataService] Workspace object migrations took 50.206083000000035ms [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncFieldMetadataService] Updating workspace metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncFieldMetadataService] Generating migrations [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncFieldMetadataService] Saving migrations [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncMetadataService] Workspace field migrations took 79.43987500000003ms [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncFieldMetadataRelationService] Updating workspace metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncFieldMetadataRelationService] Generating migrations [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncFieldMetadataRelationService] Saving migrations [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncMetadataService] Workspace relation migrations took 91.85295899999983ms [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncIndexMetadataService] Syncing index metadata [Nest] 82844 - 11/24/2025, 6:05:12 PM LOG [WorkspaceSyncMetadataService] Workspace index migrations took 153.78104199999962ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace object metadata identifiers took 130.9623330000004ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncRoleService] Syncing standard role metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace role migrations took 2.154790999999932ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncAgentService] Syncing standard agent. [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace agent migrations took 2.3623329999991256ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace migrations save took 5.95837500000016ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Executing pending migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Execute migrations took 66.94295799999963ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SyncWorkspaceMetadataCommand] Finished synchronizing workspace. [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SetStandardApplicationNotUninstallableCommand] Checking workspace applications for workspace 20202020-1c25-4d02-bf25-6aeccf7ea419 [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SetStandardApplicationNotUninstallableCommand] Successfully updated workspace application [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [UpgradeCommand] Upgrade for workspace 20202020-1c25-4d02-bf25-6aeccf7ea419 completed. PromiseMemoizer Event: A WorkspaceDataSource for workspace 20202020-1c25-4d02-bf25-6aeccf7ea419 is being cleared. Actual pool closure managed by PgPoolSharedService. Not calling dataSource.destroy(). [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [UpgradeCommand] Running command on workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db 2/2 Computing new Datasource for cacheKey: 3b8e6458-5fc1-4e63-8563-008ccddaa6db-6 out of 0 [Nest] 82844 - 11/24/2025, 6:05:13 PM DEBUG [PgPoolSharedService] Reusing existing pg Pool for key "localhost|5432|postgres||no-ssl" [Nest] 82844 - 11/24/2025, 6:05:13 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool [Nest] 82844 - 11/24/2025, 6:05:13 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool query: SELECT * FROM current_schema() [Nest] 82844 - 11/24/2025, 6:05:13 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Client acquired from pool query: SELECT version(); [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [UpgradeCommand] Upgrading workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db from=1.11.0 to=1.12.0 2/2 [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [CreateWorkspaceCustomApplicationCommand] Checking standard applications for workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [CreateWorkspaceCustomApplicationCommand] Successfully create workspace custom application [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceCustomApplicationIdNonNullableCommand] Successfully run WorkspaceCustomApplicationIdNonNullableCommand [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SyncWorkspaceMetadataCommand] Running workspace sync for workspace: 3b8e6458-5fc1-4e63-8563-008ccddaa6db (1 out of 2) [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Syncing standard objects and fields metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Syncing standard objects and fields metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncObjectMetadataService] Comparing standard objects and fields metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncObjectMetadataService] Updating workspace metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncObjectMetadataService] Generating migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncObjectMetadataService] Saving migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace object migrations took 19.26008400000046ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncFieldMetadataService] Updating workspace metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncFieldMetadataService] Generating migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncFieldMetadataService] Saving migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace field migrations took 31.371917000000394ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncFieldMetadataRelationService] Updating workspace metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncFieldMetadataRelationService] Generating migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncFieldMetadataRelationService] Saving migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace relation migrations took 36.84983300000022ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncIndexMetadataService] Syncing index metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace index migrations took 43.85666599999968ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace object metadata identifiers took 74.27345799999966ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncRoleService] Syncing standard role metadata [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace role migrations took 1.3081249999995634ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncAgentService] Syncing standard agent. [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace agent migrations took 0.7992909999993572ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Workspace migrations save took 2.1899590000002718ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Executing pending migrations [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [WorkspaceSyncMetadataService] Execute migrations took 33.817165999999816ms [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SyncWorkspaceMetadataCommand] Finished synchronizing workspace. [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SetStandardApplicationNotUninstallableCommand] Checking workspace applications for workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [SetStandardApplicationNotUninstallableCommand] Successfully updated workspace application [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [UpgradeCommand] Upgrade for workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db completed. PromiseMemoizer Event: A WorkspaceDataSource for workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db is being cleared. Actual pool closure managed by PgPoolSharedService. Not calling dataSource.destroy(). [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [UpgradeCommand] Command completed! [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [PgPoolSharedService] pg Pool for key "localhost|5432|postgres||no-ssl" has been closed. Remaining pools: 0 [Nest] 82844 - 11/24/2025, 6:05:13 PM DEBUG [PgPoolSharedService] Pool[localhost|5432|postgres||no-ssl]: Connection removed from pool [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [PgPoolSharedService] onApplicationShutdown called in PgPoolSharedService [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [PgPoolSharedModule] Shutting down PgPoolSharedModule [Nest] 82844 - 11/24/2025, 6:05:13 PM LOG [PgPoolSharedService] onApplicationShutdown called in PgPoolSharedService ——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————— NX Successfully ran target command for project twenty-server and 4 tasks it depends on (7s) With additional flags: upgrade ``` --- ...space-custom-application-id-non-nullable.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1763977334519-workspace-custom-application-id-non-nullable.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1763977334519-workspace-custom-application-id-non-nullable.ts index 0090fa851e..dc58be2b8c 100644 --- a/packages/twenty-server/src/database/typeorm/core/migrations/common/1763977334519-workspace-custom-application-id-non-nullable.ts +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1763977334519-workspace-custom-application-id-non-nullable.ts @@ -6,7 +6,11 @@ export class WorkspaceCustomApplicationIdNonNullable1763977334519 name = 'WorkspaceCustomApplicationIdNonNullable1763977334519'; public async up(queryRunner: QueryRunner): Promise { + const savepointName = 'sp_workspace_custom_application_id_non_nullable'; + try { + await queryRunner.query(`SAVEPOINT ${savepointName}`); + await queryRunner.query( `ALTER TABLE "core"."workspace" DROP CONSTRAINT "FK_3b1acb13a5dac9956d1a4b32755"`, ); @@ -16,7 +20,21 @@ export class WorkspaceCustomApplicationIdNonNullable1763977334519 await queryRunner.query( `ALTER TABLE "core"."workspace" ADD CONSTRAINT "FK_3b1acb13a5dac9956d1a4b32755" FOREIGN KEY ("workspaceCustomApplicationId") REFERENCES "core"."application"("id") ON DELETE RESTRICT ON UPDATE NO ACTION`, ); + + await queryRunner.query(`RELEASE SAVEPOINT ${savepointName}`); } catch (e) { + try { + await queryRunner.query(`ROLLBACK TO SAVEPOINT ${savepointName}`); + await queryRunner.query(`RELEASE SAVEPOINT ${savepointName}`); + } catch (rollbackError) { + // eslint-disable-next-line no-console + console.error( + 'Failed to rollback to savepoint in WorkspaceCustomApplicationIdNonNullable1763977334519', + rollbackError, + ); + throw rollbackError; + } + // eslint-disable-next-line no-console console.error( 'Swallowing WorkspaceCustomApplicationIdNonNullable1763977334519 error',