From 37b9a5538299474a27680985ac84e912de04d0e0 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Mon, 9 Feb 2026 19:09:13 +0100 Subject: [PATCH] Migrate metrics to prometheus (#17810) image --- .../typeorm/database-gauge.service.ts | 42 +++++++ .../src/database/typeorm/typeorm.module.ts | 6 +- .../admin-panel-health.service.spec.ts | 14 +-- .../__tests__/app.health.spec.ts | 2 +- .../connected-account.health.spec.ts | 8 +- .../__tests__/database.health.spec.ts | 6 +- .../__tests__/redis.health.spec.ts | 6 +- .../__tests__/worker.health.spec.ts | 6 +- .../admin-panel/admin-panel-health.service.ts | 14 +-- .../admin-panel/admin-panel.module.ts | 17 ++- .../admin-panel/admin-panel.resolver.ts | 2 +- .../health-error-messages.constants.ts | 0 .../health-indicators-timeout.conts.ts | 0 .../constants/health-indicators.constants.ts | 2 +- .../metrics-failure-rate-threshold.const.ts | 0 .../admin-panel-health-indicator.input.ts | 2 +- .../admin-panel-health-service-data.dto.ts | 2 +- .../dtos/queue-metrics-data.dto.ts | 2 +- .../admin-panel/dtos/system-health.dto.ts | 2 +- .../enums/health-indicator-id.enum.ts | 0 .../indicators/app.health.ts | 2 +- .../indicators/connected-account.health.ts | 6 +- .../indicators/database.health.ts | 6 +- .../indicators/redis.health.ts | 6 +- .../indicators/worker.health.ts | 8 +- .../types/account-sync-metrics.types.ts | 0 .../types/worker-queue-health.type.ts | 2 +- .../types/worker-queue-metrics.type.ts | 0 .../utils/health-check-timeout.util.ts | 2 +- .../utils/health-state-manager.util.ts | 0 .../billing/billing-gauge.service.ts | 117 ++++++++++++++++++ .../core-modules/billing/billing.module.ts | 4 + .../types/cache-storage-namespace.enum.ts | 1 + .../__tests__/health.controller.spec.ts | 25 ---- .../health/controllers/health.controller.ts | 43 +------ .../core-modules/health/health.module.ts | 30 +---- .../message-queue/drivers/bullmq.driver.ts | 16 ++- .../metrics/metrics-cache.service.ts | 2 +- .../core-modules/metrics/metrics.service.ts | 66 ++++++++-- .../workspace/workspace-gauge.service.ts | 78 ++++++++++++ .../workspace/workspace.module.ts | 6 +- .../subscriptions/event-stream.service.ts | 18 +-- 42 files changed, 391 insertions(+), 180 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/database-gauge.service.ts rename packages/twenty-server/src/engine/core-modules/{health/indicators => admin-panel}/__tests__/app.health.spec.ts (96%) rename packages/twenty-server/src/engine/core-modules/{health/indicators => admin-panel}/__tests__/connected-account.health.spec.ts (97%) rename packages/twenty-server/src/engine/core-modules/{health/indicators => admin-panel}/__tests__/database.health.spec.ts (95%) rename packages/twenty-server/src/engine/core-modules/{health/indicators => admin-panel}/__tests__/redis.health.spec.ts (94%) rename packages/twenty-server/src/engine/core-modules/{health/indicators => admin-panel}/__tests__/worker.health.spec.ts (97%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/constants/health-error-messages.constants.ts (100%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/constants/health-indicators-timeout.conts.ts (100%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/constants/metrics-failure-rate-threshold.const.ts (100%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/enums/health-indicator-id.enum.ts (100%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/indicators/app.health.ts (93%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/indicators/connected-account.health.ts (95%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/indicators/database.health.ts (93%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/indicators/redis.health.ts (92%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/indicators/worker.health.ts (94%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/types/account-sync-metrics.types.ts (100%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/types/worker-queue-health.type.ts (73%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/types/worker-queue-metrics.type.ts (100%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/utils/health-check-timeout.util.ts (86%) rename packages/twenty-server/src/engine/core-modules/{health => admin-panel}/utils/health-state-manager.util.ts (100%) create mode 100644 packages/twenty-server/src/engine/core-modules/billing/billing-gauge.service.ts create mode 100644 packages/twenty-server/src/engine/core-modules/workspace/workspace-gauge.service.ts diff --git a/packages/twenty-server/src/database/typeorm/database-gauge.service.ts b/packages/twenty-server/src/database/typeorm/database-gauge.service.ts new file mode 100644 index 0000000000..18e75bbd4b --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/database-gauge.service.ts @@ -0,0 +1,42 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; + +import { DataSource } from 'typeorm'; + +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; + +@Injectable() +export class DatabaseGaugeService implements OnModuleInit { + private readonly logger = new Logger(DatabaseGaugeService.name); + + constructor( + @InjectDataSource() + private readonly dataSource: DataSource, + private readonly metricsService: MetricsService, + ) {} + + onModuleInit() { + this.metricsService.createObservableGauge({ + metricName: 'twenty_database_up', + options: { + description: 'Whether the database is reachable (1 = up, 0 = down)', + }, + callback: async () => { + return this.isDatabaseUp(); + }, + cacheValue: true, + }); + } + + private async isDatabaseUp(): Promise { + try { + await this.dataSource.query('SELECT 1'); + + return 1; + } catch (error) { + this.logger.error('Database health check failed', error); + + return 0; + } + } +} diff --git a/packages/twenty-server/src/database/typeorm/typeorm.module.ts b/packages/twenty-server/src/database/typeorm/typeorm.module.ts index 6a126886bc..ee2691e651 100644 --- a/packages/twenty-server/src/database/typeorm/typeorm.module.ts +++ b/packages/twenty-server/src/database/typeorm/typeorm.module.ts @@ -2,10 +2,12 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { typeORMCoreModuleOptions } from 'src/database/typeorm/core/core.datasource'; +import { DatabaseGaugeService } from 'src/database/typeorm/database-gauge.service'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; @Module({ - imports: [TypeOrmModule.forRoot(typeORMCoreModuleOptions)], - providers: [], + imports: [TypeOrmModule.forRoot(typeORMCoreModuleOptions), MetricsModule], + providers: [DatabaseGaugeService], exports: [], }) export class TypeORMModule {} diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts index f4e8f70aac..8e7922d198 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts @@ -8,13 +8,13 @@ import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants import { type SystemHealthDTO } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto'; import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; -import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; -import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health'; -import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; -import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; -import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; +import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health'; +import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/app.health.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/app.health.spec.ts similarity index 96% rename from packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/app.health.spec.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/app.health.spec.ts index d17817656d..327bee6875 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/app.health.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/app.health.spec.ts @@ -4,7 +4,7 @@ import { getRepositoryToken } from '@nestjs/typeorm'; import { type Repository } from 'typeorm'; -import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; +import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; describe('AppHealthIndicator', () => { diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/connected-account.health.spec.ts similarity index 97% rename from packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/connected-account.health.spec.ts index 34283532b2..69f94c104c 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/connected-account.health.spec.ts @@ -1,10 +1,10 @@ import { HealthIndicatorService } from '@nestjs/terminus'; import { Test, type TestingModule } from '@nestjs/testing'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; -import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const'; -import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts'; +import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const'; +import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health'; import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/database.health.spec.ts similarity index 95% rename from packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/database.health.spec.ts index a07d195c55..7f46764cbc 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/database.health.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/database.health.spec.ts @@ -4,9 +4,9 @@ import { getDataSourceToken } from '@nestjs/typeorm'; import { type DataSource } from 'typeorm'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; -import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health'; describe('DatabaseHealthIndicator', () => { let service: DatabaseHealthIndicator; diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/redis.health.spec.ts similarity index 94% rename from packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/redis.health.spec.ts index 483ebc6578..dc01a25f1c 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/redis.health.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/redis.health.spec.ts @@ -3,9 +3,9 @@ import { Test, type TestingModule } from '@nestjs/testing'; import { type Redis } from 'ioredis'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; -import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts'; +import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health'; import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; describe('RedisHealthIndicator', () => { diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/worker.health.spec.ts similarity index 97% rename from packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/worker.health.spec.ts index ffd8c54683..400aa70ea8 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/worker.health.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/worker.health.spec.ts @@ -3,9 +3,9 @@ import { Test, type TestingModule } from '@nestjs/testing'; import { type Redis } from 'ioredis'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; -import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts index 1c971b0409..b632aa2716 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts @@ -12,13 +12,13 @@ import { type QueueMetricsDataDTO } from 'src/engine/core-modules/admin-panel/dt import { type SystemHealthDTO } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto'; import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; -import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; -import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health'; -import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; -import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; -import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; -import { type WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; +import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health'; +import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health'; +import { type WorkerQueueHealth } from 'src/engine/core-modules/admin-panel/types/worker-queue-health.type'; import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts index fdf9299082..7db521a3b0 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.module.ts @@ -12,22 +12,28 @@ import { AuthModule } from 'src/engine/core-modules/auth/auth.module'; import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { FileModule } from 'src/engine/core-modules/file/file.module'; -import { HealthModule } from 'src/engine/core-modules/health/health.module'; +import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health'; +import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health'; +import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health'; +import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health'; +import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health'; import { ImpersonationModule } from 'src/engine/core-modules/impersonation/impersonation.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; import { TelemetryModule } from 'src/engine/core-modules/telemetry/telemetry.module'; import { UserEntity } from 'src/engine/core-modules/user/user.entity'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module'; @Module({ imports: [ - TypeOrmModule.forFeature([UserEntity]), + TypeOrmModule.forFeature([UserEntity, WorkspaceEntity]), AuthModule, FileModule, WorkspaceDomainsModule, - HealthModule, RedisClientModule, TerminusModule, + MetricsModule, FeatureFlagModule, AuditModule, TelemetryModule, @@ -40,6 +46,11 @@ import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permi AdminPanelHealthService, AdminPanelQueueService, SecureHttpClientService, + DatabaseHealthIndicator, + RedisHealthIndicator, + WorkerHealthIndicator, + ConnectedAccountHealth, + AppHealthIndicator, ], exports: [AdminPanelService], }) diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts index c16aad086e..2247d807a1 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts @@ -25,7 +25,7 @@ import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/service import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter'; import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe'; import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { ConfigVariableGraphqlApiExceptionFilter } from 'src/engine/core-modules/twenty-config/filters/config-variable-graphql-api-exception.filter'; diff --git a/packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-error-messages.constants.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-error-messages.constants.ts diff --git a/packages/twenty-server/src/engine/core-modules/health/constants/health-indicators-timeout.conts.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/constants/health-indicators-timeout.conts.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts.ts diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators.constants.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators.constants.ts index e3985258c0..f2be4b991f 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators.constants.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators.constants.ts @@ -1,4 +1,4 @@ -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; type HealthIndicatorInfo = { id: HealthIndicatorId; diff --git a/packages/twenty-server/src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const.ts diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-indicator.input.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-indicator.input.ts index 8123c42ff5..696bf907f6 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-indicator.input.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-indicator.input.ts @@ -1,6 +1,6 @@ import { Field } from '@nestjs/graphql'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; export class HealthIndicatorInput { @Field(() => HealthIndicatorId) diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts index 29641800e9..7079d5751a 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts @@ -2,7 +2,7 @@ import { Field, ObjectType } from '@nestjs/graphql'; import { AdminPanelWorkerQueueHealthDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto'; import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; @ObjectType('AdminPanelHealthServiceData') export class AdminPanelHealthServiceDataDTO { diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/queue-metrics-data.dto.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/queue-metrics-data.dto.ts index c28681b30f..52c1ec093c 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/queue-metrics-data.dto.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/queue-metrics-data.dto.ts @@ -2,7 +2,7 @@ import { Field, ObjectType } from '@nestjs/graphql'; import { QueueMetricsSeriesDTO } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-series.dto'; import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum'; -import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-queue-metrics.type'; +import { WorkerQueueMetrics } from 'src/engine/core-modules/admin-panel/types/worker-queue-metrics.type'; @ObjectType('QueueMetricsData') export class QueueMetricsDataDTO { diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts index 0106bf4043..b41990936d 100644 --- a/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts @@ -1,7 +1,7 @@ import { Field, ObjectType } from '@nestjs/graphql'; import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; +import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum'; @ObjectType('SystemHealthService') export class SystemHealthServiceDTO { diff --git a/packages/twenty-server/src/engine/core-modules/health/enums/health-indicator-id.enum.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/enums/health-indicator-id.enum.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/enums/health-indicator-id.enum.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/enums/health-indicator-id.enum.ts diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/app.health.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/app.health.ts similarity index 93% rename from packages/twenty-server/src/engine/core-modules/health/indicators/app.health.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/indicators/app.health.ts index c58b46cda0..7e26567f16 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/app.health.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/app.health.ts @@ -7,7 +7,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; -import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util'; +import { HealthStateManager } from 'src/engine/core-modules/admin-panel/utils/health-state-manager.util'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; @Injectable() diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/connected-account.health.ts similarity index 95% rename from packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/indicators/connected-account.health.ts index 901ab495b3..7a1ef818ce 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/connected-account.health.ts @@ -4,9 +4,9 @@ import { HealthIndicatorService, } from '@nestjs/terminus'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const'; -import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util'; import { CALENDAR_SYNC_METRICS_BY_STATUS, MESSAGE_SYNC_METRICS_BY_STATUS, diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/database.health.ts similarity index 93% rename from packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/indicators/database.health.ts index 3511ed8992..c79982bcce 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/database.health.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/database.health.ts @@ -7,9 +7,9 @@ import { InjectDataSource } from '@nestjs/typeorm'; import { DataSource } from 'typeorm'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; -import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util'; +import { HealthStateManager } from 'src/engine/core-modules/admin-panel/utils/health-state-manager.util'; @Injectable() export class DatabaseHealthIndicator { diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/redis.health.ts similarity index 92% rename from packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/indicators/redis.health.ts index d607abfed2..84034e0d83 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/redis.health.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/redis.health.ts @@ -4,9 +4,9 @@ import { HealthIndicatorService, } from '@nestjs/terminus'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; -import { HealthStateManager } from 'src/engine/core-modules/health/utils/health-state-manager.util'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util'; +import { HealthStateManager } from 'src/engine/core-modules/admin-panel/utils/health-state-manager.util'; import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; @Injectable() diff --git a/packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/worker.health.ts similarity index 94% rename from packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/indicators/worker.health.ts index a0a32cc60a..9b97a476b9 100644 --- a/packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/indicators/worker.health.ts @@ -6,10 +6,10 @@ import { import { Queue } from 'bullmq'; -import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants'; -import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const'; -import { type WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type'; -import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util'; +import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants'; +import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const'; +import { type WorkerQueueHealth } from 'src/engine/core-modules/admin-panel/types/worker-queue-health.type'; +import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service'; diff --git a/packages/twenty-server/src/engine/core-modules/health/types/account-sync-metrics.types.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/types/account-sync-metrics.types.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/types/account-sync-metrics.types.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/types/account-sync-metrics.types.ts diff --git a/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/types/worker-queue-health.type.ts similarity index 73% rename from packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/types/worker-queue-health.type.ts index c99a6e2f0a..aaf0af81d6 100644 --- a/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/types/worker-queue-health.type.ts @@ -1,6 +1,6 @@ import { Field, ObjectType } from '@nestjs/graphql'; -import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-queue-metrics.type'; +import { WorkerQueueMetrics } from 'src/engine/core-modules/admin-panel/types/worker-queue-metrics.type'; @ObjectType() export class WorkerQueueHealth { diff --git a/packages/twenty-server/src/engine/core-modules/health/types/worker-queue-metrics.type.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/types/worker-queue-metrics.type.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/types/worker-queue-metrics.type.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/types/worker-queue-metrics.type.ts diff --git a/packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.ts similarity index 86% rename from packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.ts index 7504d61849..75d1cc86aa 100644 --- a/packages/twenty-server/src/engine/core-modules/health/utils/health-check-timeout.util.ts +++ b/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.ts @@ -1,4 +1,4 @@ -import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts'; +import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts'; export const withHealthCheckTimeout = async ( promise: Promise, diff --git a/packages/twenty-server/src/engine/core-modules/health/utils/health-state-manager.util.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-state-manager.util.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/health/utils/health-state-manager.util.ts rename to packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-state-manager.util.ts diff --git a/packages/twenty-server/src/engine/core-modules/billing/billing-gauge.service.ts b/packages/twenty-server/src/engine/core-modules/billing/billing-gauge.service.ts new file mode 100644 index 0000000000..1b52476dd4 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/billing/billing-gauge.service.ts @@ -0,0 +1,117 @@ +/* @license Enterprise */ + +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { IsNull, LessThan, Repository } from 'typeorm'; + +import { BillingSubscriptionEntity } from 'src/engine/core-modules/billing/entities/billing-subscription.entity'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; + +// Workspaces created less than 1 minute ago are excluded from the check +const WORKSPACE_AGE_THRESHOLD_MS = 60 * 1000; + +@Injectable() +export class BillingGaugeService implements OnModuleInit { + private readonly logger = new Logger(BillingGaugeService.name); + + constructor( + private readonly metricsService: MetricsService, + private readonly twentyConfigService: TwentyConfigService, + @InjectRepository(WorkspaceEntity) + private readonly workspaceRepository: Repository, + @InjectRepository(BillingSubscriptionEntity) + private readonly billingSubscriptionRepository: Repository, + ) {} + + onModuleInit() { + this.metricsService.createObservableGauge({ + metricName: 'twenty_billing_subscribed_workspaces_total', + options: { + description: 'Total number of workspaces having an active subscription', + }, + callback: async () => { + return this.getSubscribedWorkspacesCount(); + }, + cacheValue: true, + }); + + this.metricsService.createObservableGauge({ + metricName: 'twenty_billing_last_workspace_has_subscription', + options: { + description: + 'Whether the last workspace (older than 1 min) has a subscription (1 = yes, 0 = no)', + }, + callback: async () => { + return this.lastWorkspaceHasSubscription(); + }, + cacheValue: true, + }); + } + + private async getSubscribedWorkspacesCount(): Promise { + const isBillingEnabled = this.twentyConfigService.get('IS_BILLING_ENABLED'); + + if (!isBillingEnabled) { + return 0; + } + + try { + return this.billingSubscriptionRepository.count({ + where: { deletedAt: IsNull() }, + }); + } catch (error) { + this.logger.error('Failed to count subscribed workspaces', error); + + return 0; + } + } + + private async lastWorkspaceHasSubscription(): Promise { + const isBillingEnabled = this.twentyConfigService.get('IS_BILLING_ENABLED'); + + if (!isBillingEnabled) { + return 1; + } + + try { + const ageThreshold = new Date(Date.now() - WORKSPACE_AGE_THRESHOLD_MS); + + // Find the most recently created workspace that is older than 1 minute + const lastWorkspace = await this.workspaceRepository.findOne({ + where: { + deletedAt: IsNull(), + createdAt: LessThan(ageThreshold), + }, + order: { createdAt: 'DESC' }, + }); + + if (!lastWorkspace) { + return 1; + } + + const subscription = await this.billingSubscriptionRepository.findOne({ + where: { + workspaceId: lastWorkspace.id, + deletedAt: IsNull(), + }, + }); + + if (!subscription) { + this.logger.warn( + `Billing issue: workspace ${lastWorkspace.id} has no subscription`, + ); + + return 0; + } + + return 1; + } catch (error) { + this.logger.error('Failed to check last workspace subscription', error); + + return 0; + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts b/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts index 8af8cf19fa..4454d215e8 100644 --- a/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts +++ b/packages/twenty-server/src/engine/core-modules/billing/billing.module.ts @@ -3,6 +3,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { BillingGaugeService } from 'src/engine/core-modules/billing/billing-gauge.service'; import { BillingResolver } from 'src/engine/core-modules/billing/billing.resolver'; import { BillingSyncCustomerDataCommand } from 'src/engine/core-modules/billing/commands/billing-sync-customer-data.command'; import { BillingSyncPlansDataCommand } from 'src/engine/core-modules/billing/commands/billing-sync-plans-data.command'; @@ -34,6 +35,7 @@ import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { AiBillingModule } from 'src/engine/metadata-modules/ai/ai-billing/ai-billing.module'; @@ -63,6 +65,7 @@ import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permi FeatureFlagEntity, ]), DataSourceModule, + MetricsModule, ], providers: [ BillingSubscriptionService, @@ -84,6 +87,7 @@ import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permi BillingPriceService, BillingCreditRolloverService, MeteredCreditService, + BillingGaugeService, ], exports: [ BillingSubscriptionService, diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum.ts index b4725ee86b..1630028bb9 100644 --- a/packages/twenty-server/src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum.ts @@ -5,5 +5,6 @@ export enum CacheStorageNamespace { EngineWorkspace = 'engine:workspace', EngineLock = 'engine:lock', EngineHealth = 'engine:health', + EngineMetrics = 'engine:metrics', EngineSubscriptions = 'engine:subscriptions', } diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts index 1c6fc7318b..b6ff765289 100644 --- a/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts @@ -2,11 +2,6 @@ import { HealthCheckService } from '@nestjs/terminus'; import { Test, type TestingModule } from '@nestjs/testing'; import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller'; -import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; -import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health'; -import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; -import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; -import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; describe('HealthController', () => { let healthController: HealthController; @@ -19,26 +14,6 @@ describe('HealthController', () => { provide: HealthCheckService, useValue: { check: jest.fn() }, }, - { - provide: DatabaseHealthIndicator, - useValue: { isHealthy: jest.fn() }, - }, - { - provide: RedisHealthIndicator, - useValue: { isHealthy: jest.fn() }, - }, - { - provide: WorkerHealthIndicator, - useValue: { isHealthy: jest.fn() }, - }, - { - provide: ConnectedAccountHealth, - useValue: { isHealthy: jest.fn() }, - }, - { - provide: AppHealthIndicator, - useValue: { isHealthy: jest.fn() }, - }, ], }).compile(); diff --git a/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts b/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts index e0ca471e5e..9a31af1436 100644 --- a/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts @@ -1,31 +1,12 @@ -import { - BadRequestException, - Controller, - Get, - Param, - UseGuards, -} from '@nestjs/common'; +import { Controller, Get, UseGuards } from '@nestjs/common'; import { HealthCheck, HealthCheckService } from '@nestjs/terminus'; -import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum'; -import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; -import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health'; -import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health'; -import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health'; -import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health'; import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard'; import { PublicEndpointGuard } from 'src/engine/guards/public-endpoint.guard'; @Controller('healthz') export class HealthController { - constructor( - private readonly health: HealthCheckService, - private readonly databaseHealth: DatabaseHealthIndicator, - private readonly redisHealth: RedisHealthIndicator, - private readonly workerHealth: WorkerHealthIndicator, - private readonly connectedAccountHealth: ConnectedAccountHealth, - private readonly appHealth: AppHealthIndicator, - ) {} + constructor(private readonly health: HealthCheckService) {} @Get() @UseGuards(PublicEndpointGuard, NoPermissionGuard) @@ -33,24 +14,4 @@ export class HealthController { check() { return this.health.check([]); } - - @Get(':indicatorId') - @UseGuards(PublicEndpointGuard, NoPermissionGuard) - @HealthCheck() - checkService(@Param('indicatorId') indicatorId: HealthIndicatorId) { - const checks = { - [HealthIndicatorId.database]: () => this.databaseHealth.isHealthy(), - [HealthIndicatorId.redis]: () => this.redisHealth.isHealthy(), - [HealthIndicatorId.worker]: () => this.workerHealth.isHealthy(), - [HealthIndicatorId.connectedAccount]: () => - this.connectedAccountHealth.isHealthy(), - [HealthIndicatorId.app]: () => this.appHealth.isHealthy(), - }; - - if (!(indicatorId in checks)) { - throw new BadRequestException(`Invalid indicatorId: ${indicatorId}`); - } - - return this.health.check([checks[indicatorId]]); - } } diff --git a/packages/twenty-server/src/engine/core-modules/health/health.module.ts b/packages/twenty-server/src/engine/core-modules/health/health.module.ts index a1f498e890..1f28a27ba6 100644 --- a/packages/twenty-server/src/engine/core-modules/health/health.module.ts +++ b/packages/twenty-server/src/engine/core-modules/health/health.module.ts @@ -1,38 +1,10 @@ import { Module } from '@nestjs/common'; import { TerminusModule } from '@nestjs/terminus'; -import { TypeOrmModule } from '@nestjs/typeorm'; import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller'; -import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health'; -import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; -import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module'; -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { ConnectedAccountHealth } from './indicators/connected-account.health'; -import { DatabaseHealthIndicator } from './indicators/database.health'; -import { RedisHealthIndicator } from './indicators/redis.health'; -import { WorkerHealthIndicator } from './indicators/worker.health'; @Module({ - imports: [ - TerminusModule, - RedisClientModule, - TypeOrmModule.forFeature([WorkspaceEntity]), - MetricsModule, - ], + imports: [TerminusModule], controllers: [HealthController], - providers: [ - DatabaseHealthIndicator, - RedisHealthIndicator, - WorkerHealthIndicator, - ConnectedAccountHealth, - AppHealthIndicator, - ], - exports: [ - DatabaseHealthIndicator, - RedisHealthIndicator, - WorkerHealthIndicator, - ConnectedAccountHealth, - AppHealthIndicator, - ], }) export class HealthModule {} diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts index d82b30cccd..ae9bb55365 100644 --- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts @@ -52,15 +52,17 @@ export class BullMQDriver ) {} onModuleInit() { - this.metricsService.createObservableGauge( - 'twenty_queue_jobs_waiting_total', - { description: 'Current number of jobs waiting in queue' }, - async (observableResult) => { + this.metricsService.createObservableGauge({ + metricName: 'twenty_queue_jobs_waiting_total', + options: { description: 'Current number of jobs waiting in queue' }, + callback: async () => { + let totalWaiting = 0; + for (const [queueName, queue] of Object.entries(this.queueMap)) { try { const waitingCount = await queue.count(); - observableResult.observe(waitingCount, { queue: queueName }); + totalWaiting += waitingCount; } catch (error) { this.logger.error( `Failed to collect waiting jobs metrics for queue ${queueName}`, @@ -68,8 +70,10 @@ export class BullMQDriver ); } } + + return totalWaiting; }, - ); + }); } register(queueName: MessageQueue): void { diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts index 5cae093e18..b6d7624f57 100644 --- a/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics-cache.service.ts @@ -14,7 +14,7 @@ export class MetricsCacheService { private readonly healthCacheTtl: number; constructor( - @InjectCacheStorage(CacheStorageNamespace.EngineHealth) + @InjectCacheStorage(CacheStorageNamespace.EngineMetrics) private readonly cacheStorage: CacheStorageService, private readonly twentyConfigService: TwentyConfigService, ) { diff --git a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts index b6304f640a..3c9add6721 100644 --- a/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts +++ b/packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { metrics, @@ -6,30 +6,76 @@ import { type Meter, type MetricOptions, type ObservableGauge, - type ObservableResult, } from '@opentelemetry/api'; +import { isDefined } from 'twenty-shared/utils'; +import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; +import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; +import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; import { MetricsCacheService } from 'src/engine/core-modules/metrics/metrics-cache.service'; import { type MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; const METER_NAME = 'twenty-server'; +const METRICS_CACHE_TTL = 60 * 1000; // 1 minute @Injectable() export class MetricsService { - constructor(private readonly metricsCacheService: MetricsCacheService) {} + private readonly logger = new Logger(MetricsService.name); + + constructor( + private readonly metricsCacheService: MetricsCacheService, + @InjectCacheStorage(CacheStorageNamespace.EngineHealth) + private readonly healthCacheStorage: CacheStorageService, + ) {} getMeter(): Meter { return metrics.getMeter(METER_NAME); } - createObservableGauge( - name: string, - options: MetricOptions, - callback: (observableResult: ObservableResult) => void | Promise, - ): ObservableGauge { - const gauge = this.getMeter().createObservableGauge(name, options); + createObservableGauge({ + metricName, + options, + callback, + cacheValue = false, + }: { + metricName: string; + options: MetricOptions; + callback: () => number | Promise; + cacheValue?: boolean; + }): ObservableGauge { + const gauge = this.getMeter().createObservableGauge(metricName, options); - gauge.addCallback(callback); + gauge.addCallback(async (observableResult) => { + if (cacheValue) { + const cachedResult = + await this.healthCacheStorage.get(metricName); + + if (isDefined(cachedResult)) { + observableResult.observe(cachedResult); + + return; + } + } + + try { + const result = await callback(); + + observableResult.observe(result); + + if (cacheValue) { + await this.healthCacheStorage.set( + metricName, + result, + METRICS_CACHE_TTL, + ); + } + } catch (error) { + this.logger.error( + `Failed to collect gauge metric ${metricName}`, + error, + ); + } + }); return gauge; } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace-gauge.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace-gauge.service.ts new file mode 100644 index 0000000000..2b02fb104b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace-gauge.service.ts @@ -0,0 +1,78 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { IsNull, Not, Repository } from 'typeorm'; +import { WorkspaceActivationStatus } from 'twenty-shared/workspace'; + +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; + +@Injectable() +export class WorkspaceGaugeService implements OnModuleInit { + private readonly logger = new Logger(WorkspaceGaugeService.name); + + constructor( + private readonly metricsService: MetricsService, + @InjectRepository(WorkspaceEntity) + private readonly workspaceRepository: Repository, + ) {} + + onModuleInit() { + for (const status of Object.values(WorkspaceActivationStatus)) { + this.metricsService.createObservableGauge({ + metricName: `twenty_workspaces_by_status_${status.toLowerCase()}`, + options: { + description: `Number of workspaces with activation status ${status}`, + }, + callback: async () => { + return this.getWorkspaceCountByStatus(status); + }, + cacheValue: true, + }); + } + + this.metricsService.createObservableGauge({ + metricName: 'twenty_workspaces_deleted_total', + options: { + description: 'Total number of soft-deleted workspaces', + }, + callback: async () => { + return this.getDeletedWorkspacesCount(); + }, + cacheValue: true, + }); + } + + private async getWorkspaceCountByStatus( + status: WorkspaceActivationStatus, + ): Promise { + try { + return this.workspaceRepository.count({ + where: { + activationStatus: status, + deletedAt: IsNull(), + }, + }); + } catch (error) { + this.logger.error( + `Failed to count workspaces with status ${status}`, + error, + ); + + return 0; + } + } + + private async getDeletedWorkspacesCount(): Promise { + try { + return this.workspaceRepository.count({ + where: { deletedAt: Not(IsNull()) }, + withDeleted: true, + }); + } catch (error) { + this.logger.error('Failed to count deleted workspaces', error); + + return 0; + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index edb5df1cf8..9df5f169bf 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -6,6 +6,7 @@ import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { ApplicationModule } from 'src/engine/core-modules/application/application.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { AuditModule } from 'src/engine/core-modules/audit/audit.module'; import { TokenModule } from 'src/engine/core-modules/auth/token/token.module'; import { BillingModule } from 'src/engine/core-modules/billing/billing.module'; @@ -27,6 +28,7 @@ import { CheckCustomDomainValidRecordsCronJob } from 'src/engine/core-modules/wo import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { workspaceAutoResolverOpts } from 'src/engine/core-modules/workspace/workspace.auto-resolver-opts'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceGaugeService } from 'src/engine/core-modules/workspace/workspace-gauge.service'; import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver'; import { AiAgentModule } from 'src/engine/metadata-modules/ai/ai-agent/ai-agent.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; @@ -42,7 +44,8 @@ import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-m @Module({ imports: [ TypeORMModule, - TypeOrmModule.forFeature([BillingSubscriptionEntity]), + TypeOrmModule.forFeature([BillingSubscriptionEntity, WorkspaceEntity]), + MetricsModule, NestjsQueryGraphQLModule.forFeature({ imports: [ AuditModule, @@ -84,6 +87,7 @@ import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-m providers: [ WorkspaceResolver, WorkspaceService, + WorkspaceGaugeService, CheckCustomDomainValidRecordsCronCommand, CheckCustomDomainValidRecordsCronJob, ], diff --git a/packages/twenty-server/src/engine/subscriptions/event-stream.service.ts b/packages/twenty-server/src/engine/subscriptions/event-stream.service.ts index 57a6641843..ed4496d7e1 100644 --- a/packages/twenty-server/src/engine/subscriptions/event-stream.service.ts +++ b/packages/twenty-server/src/engine/subscriptions/event-stream.service.ts @@ -29,19 +29,13 @@ export class EventStreamService implements OnModuleInit { ) {} onModuleInit() { - this.metricsService.createObservableGauge( - 'twenty_event_streams_live_total', - { description: 'Current number of live event streams' }, - async (observableResult) => { - try { - const count = await this.getTotalActiveStreamCount(); - - observableResult.observe(count); - } catch (error) { - this.logger.error('Failed to collect event streams metrics', error); - } + this.metricsService.createObservableGauge({ + metricName: 'twenty_event_streams_live_total', + options: { description: 'Current number of live event streams' }, + callback: async () => { + return this.getTotalActiveStreamCount(); }, - ); + }); } async getTotalActiveStreamCount(): Promise {