Thomas Trompette
96a2456367
Monitoring for the soft-ref migration. The `workflow` /
`workflowVersion` dual-write into core is **best-effort** (async, not
transactional; failures only go to Sentry), so `core.workflow` /
`core.workflowVersion` can silently drift from the workspace source of
truth. This adds a periodic job that detects that drift across **all
three workflow entities** and emits it as metrics.
Supersedes the earlier inline shadow-parity approach that lived on this
branch — that only covered trigger dispatch and added a cache read +
diff to every cron tick and every DB-event batch (too much hot-path
overhead). This is broader and fully off the dispatch path.
## What
A cron (`cron:workflow:core-consistency-check`, every 3 hours, wired
into `cron:register:all`). Per run:
- **Bounded**: `SELECT DISTINCT "workspaceId" FROM core."workflow"` —
only workspaces that actually use workflows (skips the large majority).
- Per such workspace, emit a drift metric per `(entity, driftType)` —
detect-only, plus a triage log:
- **workflow** and **workflowVersion** — `unlinked` / `missingCore` /
`orphanCore` / `fieldMismatch`, via cross-schema `COUNT` aggregates
(core + workspace are the same DB, so indexed joins — no rows pulled
into JS).
- **automated triggers** — the `workflowAutomatedTrigger` table vs the
`workflowAutomatedTriggerMaps` cache: `inTableNotCache` /
`inCacheNotTable` / settings `mismatch`.
- Per-workspace failures are isolated (caught → Sentry) so one bad
workspace does not stop the sweep.
## Why it is efficient
Two central `core.*` queries + a few `COUNT` queries per
*workflow-using* workspace, on a relaxed cadence, off the dispatch path.
Shardable across ticks later if needed.
## Metrics
`workflow-core-consistency/{workflow,version,automated-trigger}/drift`
counters, attribute `driftType`. Dashboards: twentyhq/twenty-infra#800.
## Not in scope
Detect-only — no auto-heal (the existing backfill/rebuild command can
heal). No dispatch or flag changes.
## Test
- The consistency SQL (workflow/version sync counts, orphan counts,
trigger read) validated against a live workspace (it surfaced real drift
there — unlinked versions + an orphan core version). The whole
cron→service→SQL→metric pipeline is proven live: the cron is already
emitting real drift counters on a running server.
- Unit specs for the service (clean → no metric; per-entity drift per
dimension; per-workspace error isolation).
- Command boots and registers via `cron:register:all` (verified).
Typecheck + lint clean.
119 lines
7.4 KiB
TypeScript
119 lines
7.4 KiB
TypeScript
import { Module } from '@nestjs/common';
|
|
import { TypeOrmModule } from '@nestjs/typeorm';
|
|
|
|
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
|
|
import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.command';
|
|
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
|
|
import { SecretEncryptionRotationModule } from 'src/database/commands/secret-encryption-rotation/secret-encryption-rotation.module';
|
|
import { GenerateInstanceCommandCommand } from 'src/database/commands/generate-instance-command.command';
|
|
import { InstallPreInstalledAppsCommand } from 'src/database/commands/install-pre-installed-apps.command';
|
|
import { InstanceCommandGenerationService } from 'src/database/commands/instance-command-generation.service';
|
|
import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command';
|
|
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
|
|
import { RebuildApplicationDefaultDepsCommand } from 'src/database/commands/rebuild-application-default-deps.command';
|
|
import { RunInstanceCommandsCommand } from 'src/database/commands/run-instance-commands.command';
|
|
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
|
|
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
|
|
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
|
|
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
|
|
import { GenerateApiKeyCommand } from 'src/engine/core-modules/api-key/commands/generate-api-key.command';
|
|
import { MarketplaceModule } from 'src/engine/core-modules/application/application-marketplace/marketplace.module';
|
|
import { StaleRegistrationCleanupModule } from 'src/engine/core-modules/application/application-oauth/stale-registration-cleanup/stale-registration-cleanup.module';
|
|
import { ApplicationUpgradeModule } from 'src/engine/core-modules/application/application-upgrade/application-upgrade.module';
|
|
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
|
|
import { PreInstalledAppsModule } from 'src/engine/core-modules/application/pre-installed-apps/pre-installed-apps.module';
|
|
import { BillingReminderModule } from 'src/engine/core-modules/billing/reminders/billing-reminder.module';
|
|
import { EnterpriseKeyValidationCronCommand } from 'src/engine/core-modules/enterprise/cron/command/enterprise-key-validation.cron.command';
|
|
import { EnterpriseModule } from 'src/engine/core-modules/enterprise/enterprise.module';
|
|
import { EventLogCleanupModule } from 'src/engine/core-modules/event-logs/cleanup/event-log-cleanup.module';
|
|
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
|
import { RotateSigningKeysCronCommand } from 'src/engine/core-modules/jwt/crons/commands/rotate-signing-keys.cron.command';
|
|
import { FileModule } from 'src/engine/core-modules/file/file.module';
|
|
import { PublicDomainModule } from 'src/engine/core-modules/public-domain/public-domain.module';
|
|
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
|
|
import { UpgradeStatusCommand } from 'src/engine/core-modules/upgrade/commands/upgrade-status.command';
|
|
import { UpgradeModule } from 'src/engine/core-modules/upgrade/upgrade.module';
|
|
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
|
import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module';
|
|
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
|
|
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
|
|
import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
|
|
import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/provide-workspace-scoped-repository';
|
|
import { CodeInterpreterSessionCleanupModule } from 'src/engine/core-modules/code-interpreter/crons/code-interpreter-session-cleanup.module';
|
|
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
|
|
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
|
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
|
import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
|
|
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
|
|
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
|
import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace-migration/workspace-migration.module';
|
|
import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
|
|
import { WebhookSubscriptionModule } from 'src/modules/connected-account/webhook-subscription-manager/webhook-subscription.module';
|
|
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
|
|
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
|
|
import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module';
|
|
import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/automated-trigger/automated-trigger.module';
|
|
import { WorkflowCoreConsistencyModule } from 'src/modules/workflow/workflow-core-consistency/workflow-core-consistency.module';
|
|
|
|
@Module({
|
|
imports: [
|
|
UpgradeVersionCommandModule,
|
|
TypeOrmModule.forFeature([WorkspaceEntity, RoleEntity]),
|
|
WorkspaceExportModule,
|
|
// Cron command dependencies
|
|
MessagingImportManagerModule,
|
|
CalendarEventImportManagerModule,
|
|
WebhookSubscriptionModule,
|
|
AutomatedTriggerModule,
|
|
WorkflowCoreConsistencyModule,
|
|
FileModule,
|
|
WorkspaceModule,
|
|
WorkflowRunQueueModule,
|
|
// Data seeding dependencies
|
|
TypeORMModule,
|
|
FieldMetadataModule,
|
|
ObjectMetadataModule,
|
|
DevSeederModule,
|
|
WorkspaceManagerModule,
|
|
WorkspaceCacheStorageModule,
|
|
ApiKeyModule,
|
|
FeatureFlagModule,
|
|
WorkspaceCleanerModule,
|
|
WorkspaceMigrationModule,
|
|
TrashCleanupModule,
|
|
BillingReminderModule,
|
|
CodeInterpreterSessionCleanupModule,
|
|
PublicDomainModule,
|
|
EventLogCleanupModule,
|
|
EnterpriseModule,
|
|
TwentyConfigModule,
|
|
MarketplaceModule,
|
|
ApplicationUpgradeModule,
|
|
StaleRegistrationCleanupModule,
|
|
PreInstalledAppsModule,
|
|
WorkspaceIteratorModule,
|
|
ApplicationModule,
|
|
WorkspaceCacheModule,
|
|
WorkspaceVersionModule,
|
|
UpgradeModule,
|
|
SecretEncryptionRotationModule,
|
|
],
|
|
providers: [
|
|
DataSeedWorkspaceCommand,
|
|
ConfirmationQuestion,
|
|
CronRegisterAllCommand,
|
|
GenerateInstanceCommandCommand,
|
|
InstanceCommandGenerationService,
|
|
RunInstanceCommandsCommand,
|
|
ListOrphanedWorkspaceEntitiesCommand,
|
|
EnterpriseKeyValidationCronCommand,
|
|
RotateSigningKeysCronCommand,
|
|
GenerateApiKeyCommand,
|
|
UpgradeStatusCommand,
|
|
RebuildApplicationDefaultDepsCommand,
|
|
InstallPreInstalledAppsCommand,
|
|
provideWorkspaceScopedRepository(RoleEntity),
|
|
],
|
|
})
|
|
export class DatabaseCommandModule {}
|