ca1571676c
## Summary Prod 2.5 upgrade failed on the slow instance command `EncryptApplicationVariableSlowInstanceCommand`: ``` [Nest] LOG [InstanceCommandRunnerService] 2.5.0_EncryptApplicationVariableSlowInstanceCommand_1798000005000 starting data migration... [Nest] WARN [SecretEncryptionService] Decrypted a legacy unprefixed AES-CTR ciphertext... [Nest] ERROR [InstanceCommandRunnerService] data migration failed TypeError: Invalid initialization vector ``` ### Root cause The migration assumes every row matching `isSecret = true AND value <> '' AND value NOT LIKE 'enc:v2:%'` is legacy AES-CTR ciphertext. In prod we found multiple `isSecret = true` rows whose `value` is plaintext (e.g. `SLACK_HOOK_URL = 'https://hooks.slack.com/services/...'`) — most likely the result of `isSecret` being flipped to true on a row that already held a plaintext value, or a write path that bypassed `ApplicationVariableEntityService.update`. Those values can't decode into the 16-byte IV that AES-CTR needs, so `Buffer.from(value, 'base64')` truncates at the first non-base64 char (`:`), the buffer is < 16 bytes, and `createDecipheriv` throws. ### Fix Follow the same policy as `EncryptConnectedAccountTokensSlowInstanceCommand`: anything that isn't already in the `enc:v2:` envelope is plaintext. Concretely: 1. Try `decryptVersioned` — legacy CTR rows decrypt fine. 2. If it throws (mis-classified plaintext), log a warning naming the row id and fall back to treating `row.value` as plaintext. 3. Encrypt the resulting plaintext into the `enc:v2:` envelope and update the row. In-loop `isSecret` guard is kept (alongside the SQL filter) so non-secret rows are never touched even if the SQL filter is ever loosened. ### Integration test coverage Added one new case alongside the existing ones in `…encrypt-application-variable.integration-spec.ts`: - `treats plaintext-under-isSecret=true as plaintext and re-encrypts as v2` — seeds a row with `isSecret = true` and a URL value (`:` and `/` are not base64, so this is the exact failure shape from prod), runs the migration, and asserts the value is now `enc:v2:...` and decrypts back to the original URL. Existing cases unchanged: legacy CTR happy path, non-secret rows untouched, idempotent across re-runs, `up()` adds the CHECK constraint, `down()` removes it. ### Why this is a 2-5 edit `TWENTY_CURRENT_VERSION` is now 2.6.0, so editing a 2-5 file trips the `server-previous-version-upgrade-mutation-guard` — `ci:allow-previous-version-upgrade-mutation` label is on the PR. `up()` and `down()` are unchanged; only `runDataMigration` is modified. ## Test plan - [ ] Re-deploy 2.5 to prod and confirm `EncryptApplicationVariableSlowInstanceCommand` completes - [ ] Inspect warning log to count rows that went through the plaintext fallback - [ ] Verify resulting secret rows all satisfy `value = '' OR value LIKE 'enc:v2:%'` and the CHECK constraint is in place
265 lines
7.9 KiB
TypeScript
265 lines
7.9 KiB
TypeScript
import crypto from 'crypto';
|
|
|
|
import { config } from 'dotenv';
|
|
import { isDefined } from 'twenty-shared/utils';
|
|
import { DataSource } from 'typeorm';
|
|
|
|
import { buildSecretEncryptionServiceFromEnv } from 'test/integration/upgrade/utils/build-secret-encryption-service.util';
|
|
|
|
import { SECRET_ENCRYPTION_ENVELOPE_V2_PREFIX } from 'src/engine/core-modules/secret-encryption/constants/secret-encryption.constant';
|
|
import { SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
|
|
|
|
import { EncryptApplicationVariableSlowInstanceCommand } from 'src/database/commands/upgrade-version-command/2-5/2-5-instance-command-slow-1798000005000-encrypt-application-variable';
|
|
|
|
jest.useRealTimers();
|
|
|
|
config({
|
|
path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
|
|
override: true,
|
|
});
|
|
|
|
const TEST_ROW_KEY_PREFIX = 'ENCRYPT_APP_VAR_TEST_';
|
|
const CHECK_CONSTRAINT_NAME = 'CHK_applicationVariable_value_encrypted';
|
|
const CHECK_CONSTRAINT_EXPR = `"isSecret" = false OR "value" = '' OR "value" LIKE '${SECRET_ENCRYPTION_ENVELOPE_V2_PREFIX}%'`;
|
|
|
|
const dropCheckConstraint = (dataSource: DataSource): Promise<unknown> =>
|
|
dataSource.query(
|
|
`ALTER TABLE "core"."applicationVariable"
|
|
DROP CONSTRAINT IF EXISTS "${CHECK_CONSTRAINT_NAME}"`,
|
|
);
|
|
|
|
const restoreCheckConstraint = async (
|
|
dataSource: DataSource,
|
|
): Promise<void> => {
|
|
await dropCheckConstraint(dataSource);
|
|
await dataSource.query(
|
|
`ALTER TABLE "core"."applicationVariable"
|
|
ADD CONSTRAINT "${CHECK_CONSTRAINT_NAME}"
|
|
CHECK (${CHECK_CONSTRAINT_EXPR})`,
|
|
);
|
|
};
|
|
|
|
describe('2-5 slow instance command 1798000005000 - EncryptApplicationVariableSlowInstanceCommand (integration)', () => {
|
|
let dataSource: DataSource;
|
|
let secretEncryptionService: SecretEncryptionService;
|
|
let command: EncryptApplicationVariableSlowInstanceCommand;
|
|
let workspaceId: string;
|
|
let applicationId: string;
|
|
const seededRowIds: string[] = [];
|
|
|
|
const seedRow = async ({
|
|
isSecret,
|
|
value,
|
|
}: {
|
|
isSecret: boolean;
|
|
value: string;
|
|
}): Promise<string> => {
|
|
await dropCheckConstraint(dataSource);
|
|
|
|
const id = crypto.randomUUID();
|
|
const universalIdentifier = crypto.randomUUID();
|
|
const key = `${TEST_ROW_KEY_PREFIX}${id}`;
|
|
|
|
await dataSource.query(
|
|
`INSERT INTO "core"."applicationVariable"
|
|
(id, "universalIdentifier", "applicationId", "workspaceId",
|
|
"key", "value", "isSecret")
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
|
|
[
|
|
id,
|
|
universalIdentifier,
|
|
applicationId,
|
|
workspaceId,
|
|
key,
|
|
value,
|
|
isSecret,
|
|
],
|
|
);
|
|
|
|
seededRowIds.push(id);
|
|
|
|
return id;
|
|
};
|
|
|
|
beforeAll(async () => {
|
|
dataSource = new DataSource({
|
|
type: 'postgres',
|
|
url: process.env.PG_DATABASE_URL,
|
|
schema: 'core',
|
|
entities: [],
|
|
synchronize: false,
|
|
});
|
|
await dataSource.initialize();
|
|
|
|
secretEncryptionService = buildSecretEncryptionServiceFromEnv();
|
|
command = new EncryptApplicationVariableSlowInstanceCommand(
|
|
secretEncryptionService,
|
|
);
|
|
|
|
const [seedWorkspace] = await dataSource.query(
|
|
`SELECT id, "workspaceCustomApplicationId"
|
|
FROM "core"."workspace"
|
|
WHERE "workspaceCustomApplicationId" IS NOT NULL
|
|
LIMIT 1`,
|
|
);
|
|
|
|
if (!isDefined(seedWorkspace)) {
|
|
throw new Error(
|
|
'No seeded workspace with a custom application found; run database:reset before the integration suite.',
|
|
);
|
|
}
|
|
|
|
workspaceId = seedWorkspace.id as string;
|
|
applicationId = seedWorkspace.workspaceCustomApplicationId as string;
|
|
}, 30000);
|
|
|
|
afterEach(async () => {
|
|
if (seededRowIds.length > 0) {
|
|
await dataSource.query(
|
|
`DELETE FROM "core"."applicationVariable" WHERE id = ANY($1::uuid[])`,
|
|
[seededRowIds],
|
|
);
|
|
seededRowIds.length = 0;
|
|
}
|
|
await restoreCheckConstraint(dataSource);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await dataSource?.destroy();
|
|
});
|
|
|
|
it('upgrades legacy CTR secret rows to enc:v2 with workspaceId-bound HKDF', async () => {
|
|
const plaintext = 'legacy-ctr-application-variable-secret';
|
|
const id = await seedRow({
|
|
isSecret: true,
|
|
value: secretEncryptionService.encrypt(plaintext),
|
|
});
|
|
|
|
await command.runDataMigration(dataSource);
|
|
|
|
const [row] = await dataSource.query(
|
|
`SELECT "value" FROM "core"."applicationVariable" WHERE id = $1`,
|
|
[id],
|
|
);
|
|
|
|
expect(row.value.startsWith(SECRET_ENCRYPTION_ENVELOPE_V2_PREFIX)).toBe(
|
|
true,
|
|
);
|
|
expect(
|
|
secretEncryptionService.decryptVersioned(row.value, { workspaceId }),
|
|
).toBe(plaintext);
|
|
});
|
|
|
|
it('leaves non-secret rows untouched', async () => {
|
|
const plaintext = 'https://public.example.com/manifest.json';
|
|
const id = await seedRow({ isSecret: false, value: plaintext });
|
|
|
|
await command.runDataMigration(dataSource);
|
|
|
|
const [row] = await dataSource.query(
|
|
`SELECT "value" FROM "core"."applicationVariable" WHERE id = $1`,
|
|
[id],
|
|
);
|
|
|
|
expect(row.value).toBe(plaintext);
|
|
});
|
|
|
|
it('treats plaintext-under-isSecret=true as plaintext and re-encrypts as v2', async () => {
|
|
const plaintext =
|
|
'https://hooks.slack.com/services/T09QGPB2ZP1/B09QUQ5LY2Z/abc';
|
|
const id = await seedRow({ isSecret: true, value: plaintext });
|
|
|
|
await command.runDataMigration(dataSource);
|
|
|
|
const [row] = await dataSource.query(
|
|
`SELECT "value" FROM "core"."applicationVariable" WHERE id = $1`,
|
|
[id],
|
|
);
|
|
|
|
expect(row.value.startsWith(SECRET_ENCRYPTION_ENVELOPE_V2_PREFIX)).toBe(
|
|
true,
|
|
);
|
|
expect(
|
|
secretEncryptionService.decryptVersioned(row.value, { workspaceId }),
|
|
).toBe(plaintext);
|
|
});
|
|
|
|
it('leaves enc:v2 rows untouched and is idempotent across re-runs', async () => {
|
|
const plaintext = 'already-v2-secret';
|
|
const preexistingV2 = secretEncryptionService.encryptVersioned(plaintext, {
|
|
workspaceId,
|
|
});
|
|
const id = await seedRow({ isSecret: true, value: preexistingV2 });
|
|
|
|
await command.runDataMigration(dataSource);
|
|
const [afterFirstRun] = await dataSource.query(
|
|
`SELECT "value" FROM "core"."applicationVariable" WHERE id = $1`,
|
|
[id],
|
|
);
|
|
|
|
expect(afterFirstRun.value).toBe(preexistingV2);
|
|
|
|
await command.runDataMigration(dataSource);
|
|
const [afterSecondRun] = await dataSource.query(
|
|
`SELECT "value" FROM "core"."applicationVariable" WHERE id = $1`,
|
|
[id],
|
|
);
|
|
|
|
expect(afterSecondRun.value).toBe(preexistingV2);
|
|
});
|
|
|
|
it('up() applies the CHECK constraint that rejects plaintext secret inserts', async () => {
|
|
await dropCheckConstraint(dataSource);
|
|
|
|
const queryRunner = dataSource.createQueryRunner();
|
|
|
|
try {
|
|
await command.up(queryRunner);
|
|
|
|
const id = crypto.randomUUID();
|
|
|
|
seededRowIds.push(id);
|
|
|
|
await expect(
|
|
dataSource.query(
|
|
`INSERT INTO "core"."applicationVariable"
|
|
(id, "universalIdentifier", "applicationId", "workspaceId",
|
|
"key", "value", "isSecret")
|
|
VALUES ($1, $2, $3, $4, $5, 'plaintext-should-be-rejected', true)`,
|
|
[
|
|
id,
|
|
crypto.randomUUID(),
|
|
applicationId,
|
|
workspaceId,
|
|
`${TEST_ROW_KEY_PREFIX}${id}`,
|
|
],
|
|
),
|
|
).rejects.toThrow(/check constraint/i);
|
|
} finally {
|
|
await queryRunner.release();
|
|
}
|
|
});
|
|
|
|
it('down() removes the CHECK constraint and lets plaintext through (for rollback safety only)', async () => {
|
|
const queryRunner = dataSource.createQueryRunner();
|
|
|
|
try {
|
|
await command.down(queryRunner);
|
|
|
|
const id = await seedRow({
|
|
isSecret: true,
|
|
value: 'plaintext-allowed-after-down',
|
|
});
|
|
|
|
const [row] = await dataSource.query(
|
|
`SELECT "value" FROM "core"."applicationVariable" WHERE id = $1`,
|
|
[id],
|
|
);
|
|
|
|
expect(row.value).toBe('plaintext-allowed-after-down');
|
|
} finally {
|
|
await queryRunner.release();
|
|
}
|
|
});
|
|
});
|