Migrate metrics to prometheus (#17810)
<img width="1316" height="611" alt="image" src="https://github.com/user-attachments/assets/277a63ed-2a8b-41ff-be78-281de8891579" />
This commit is contained in:
+7
-7
@@ -8,13 +8,13 @@ import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants
|
||||
import { type SystemHealthDTO } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
|
||||
import { type Repository } from 'typeorm';
|
||||
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
|
||||
describe('AppHealthIndicator', () => {
|
||||
let service: AppHealthIndicator;
|
||||
let workspaceRepository: jest.Mocked<Repository<WorkspaceEntity>>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
workspaceRepository = {
|
||||
count: jest.fn(),
|
||||
} as any;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockReturnValue({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
app: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((data) => ({
|
||||
app: { status: 'down', ...data },
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
AppHealthIndicator,
|
||||
{
|
||||
provide: getRepositoryToken(WorkspaceEntity),
|
||||
useValue: workspaceRepository,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<AppHealthIndicator>(AppHealthIndicator);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return up status when no issues and no pending migrations', async () => {
|
||||
workspaceRepository.count.mockResolvedValue(2);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.app.status).toBe('up');
|
||||
expect(result.app.details.overview.totalWorkspacesCount).toBe(2);
|
||||
expect(result.app.details.overview.erroredWorkspaceCount).toBe(0);
|
||||
expect(result.app.details.system.nodeVersion).toBeDefined();
|
||||
expect(result.app.details.system.timestamp).toBeDefined();
|
||||
});
|
||||
|
||||
it('should maintain state history across health checks', async () => {
|
||||
// First check - healthy state
|
||||
workspaceRepository.count.mockResolvedValue(2);
|
||||
|
||||
await service.isHealthy();
|
||||
|
||||
// Second check - error state
|
||||
workspaceRepository.count.mockRejectedValue(
|
||||
new Error('Database connection failed'),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.app.details.stateHistory).toBeDefined();
|
||||
expect(result.app.details.stateHistory.age).toBeDefined();
|
||||
expect(result.app.details.stateHistory.timestamp).toBeDefined();
|
||||
expect(result.app.details.stateHistory.details).toBeDefined();
|
||||
});
|
||||
});
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts';
|
||||
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
|
||||
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
|
||||
describe('ConnectedAccountHealth', () => {
|
||||
let service: ConnectedAccountHealth;
|
||||
let metricsService: jest.Mocked<MetricsService>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
metricsService = {
|
||||
groupMetrics: jest.fn(),
|
||||
} as any;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockImplementation((key) => ({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
[key]: {
|
||||
status: 'up',
|
||||
details: data.details,
|
||||
},
|
||||
})),
|
||||
down: jest.fn().mockImplementation((data) => ({
|
||||
[key]: {
|
||||
status: 'down',
|
||||
error: data.error,
|
||||
details: data.details,
|
||||
},
|
||||
})),
|
||||
})),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
ConnectedAccountHealth,
|
||||
{
|
||||
provide: MetricsService,
|
||||
useValue: metricsService,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<ConnectedAccountHealth>(ConnectedAccountHealth);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe('message sync health', () => {
|
||||
it('should return up status when no message sync jobs are present', async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 0,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 0,
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('up');
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('up');
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.totalJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.failedJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.failureRate,
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it(`should return down status when message sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.messageSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(
|
||||
result.connectedAccount.details.messageSync.details.failureRate,
|
||||
).toBe(40);
|
||||
});
|
||||
});
|
||||
|
||||
describe('calendar sync health', () => {
|
||||
it('should return up status when no calendar sync jobs are present', async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 0,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 0,
|
||||
[CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 0,
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('up');
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('up');
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.totalJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.failedJobs,
|
||||
).toBe(0);
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.failureRate,
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it(`should return down status when calendar sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
[CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.calendarSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
|
||||
);
|
||||
expect(
|
||||
result.connectedAccount.details.calendarSync.details.failureRate,
|
||||
).toBe(40);
|
||||
});
|
||||
});
|
||||
|
||||
describe('timeout handling', () => {
|
||||
it('should handle message sync timeout', async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce(
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
});
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.messageSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle calendar sync timeout', async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
})
|
||||
.mockResolvedValueOnce(
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
|
||||
);
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.calendarSync.error).toBe(
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('combined health check', () => {
|
||||
it('should return combined status with both checks healthy', async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 8,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 1,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 8,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 1,
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('up');
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('up');
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('up');
|
||||
});
|
||||
|
||||
it('should return down status when both syncs fail', async () => {
|
||||
metricsService.groupMetrics
|
||||
.mockResolvedValueOnce({
|
||||
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[MessageChannelSyncStatus.ACTIVE]: 1,
|
||||
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
|
||||
[CalendarChannelSyncStatus.ACTIVE]: 1,
|
||||
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 2,
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.connectedAccount.status).toBe('down');
|
||||
expect(result.connectedAccount.error).toBe(
|
||||
`${HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE} and ${HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE}`,
|
||||
);
|
||||
expect(result.connectedAccount.details.messageSync.status).toBe('down');
|
||||
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
|
||||
});
|
||||
});
|
||||
});
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
import { getDataSourceToken } from '@nestjs/typeorm';
|
||||
|
||||
import { type DataSource } from 'typeorm';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health';
|
||||
|
||||
describe('DatabaseHealthIndicator', () => {
|
||||
let service: DatabaseHealthIndicator;
|
||||
let dataSource: jest.Mocked<DataSource>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
dataSource = {
|
||||
query: jest.fn(),
|
||||
} as any;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockReturnValue({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
database: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((data) => ({
|
||||
database: { status: 'down', ...data },
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
DatabaseHealthIndicator,
|
||||
{
|
||||
provide: getDataSourceToken(),
|
||||
useValue: dataSource,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<DatabaseHealthIndicator>(DatabaseHealthIndicator);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return up status with details when database responds', async () => {
|
||||
const mockResponses = [
|
||||
[{ version: 'PostgreSQL 15.6' }],
|
||||
[{ count: '5' }],
|
||||
[{ max_connections: '100' }],
|
||||
[{ uptime: '3600' }],
|
||||
[{ size: '1 GB' }],
|
||||
[{ table_stats: [] }],
|
||||
[{ ratio: '95.5' }],
|
||||
[{ deadlocks: '0' }],
|
||||
[{ count: '0' }],
|
||||
];
|
||||
|
||||
mockResponses.forEach((response) => {
|
||||
dataSource.query.mockResolvedValueOnce(response);
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.database.status).toBe('up');
|
||||
expect(result.database.details.system.version).toBe('PostgreSQL 15.6');
|
||||
expect(result.database.details.system.timestamp).toBeDefined();
|
||||
expect(result.database.details.connections).toEqual({
|
||||
active: 5,
|
||||
max: 100,
|
||||
utilizationPercent: 5,
|
||||
});
|
||||
expect(result.database.details.performance).toEqual({
|
||||
cacheHitRatio: '96%',
|
||||
deadlocks: 0,
|
||||
slowQueries: 0,
|
||||
});
|
||||
expect(result.database.details.databaseSize).toBe('1 GB');
|
||||
expect(result.database.details.top10Tables).toEqual([{ table_stats: [] }]);
|
||||
});
|
||||
|
||||
it('should return down status when database fails', async () => {
|
||||
dataSource.query.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.database.status).toBe('down');
|
||||
expect(result.database.message).toBe(
|
||||
HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED,
|
||||
);
|
||||
expect(result.database.details.system.timestamp).toBeDefined();
|
||||
expect(result.database.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should timeout after specified duration', async () => {
|
||||
dataSource.query.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.database.status).toBe('down');
|
||||
expect(result.database.message).toBe(
|
||||
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
|
||||
);
|
||||
expect(result.database.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should maintain state history across health checks', async () => {
|
||||
// First check - healthy state
|
||||
const mockResponses = [
|
||||
[{ version: 'PostgreSQL 15.6' }],
|
||||
[{ count: '5' }],
|
||||
[{ max_connections: '100' }],
|
||||
[{ uptime: '3600' }],
|
||||
[{ size: '1 GB' }],
|
||||
[{ table_stats: [] }],
|
||||
[{ ratio: '95.5' }],
|
||||
[{ deadlocks: '0' }],
|
||||
[{ count: '0' }],
|
||||
];
|
||||
|
||||
mockResponses.forEach((response) => {
|
||||
dataSource.query.mockResolvedValueOnce(response);
|
||||
});
|
||||
|
||||
const firstResult = await service.isHealthy();
|
||||
|
||||
expect(firstResult.database.status).toBe('up');
|
||||
|
||||
// Second check - error state
|
||||
dataSource.query.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.database.details.stateHistory).toMatchObject({
|
||||
age: expect.any(Number),
|
||||
timestamp: expect.any(Date),
|
||||
details: {
|
||||
system: {
|
||||
version: 'PostgreSQL 15.6',
|
||||
timestamp: expect.any(String),
|
||||
uptime: expect.any(String),
|
||||
},
|
||||
connections: {
|
||||
active: 5,
|
||||
max: 100,
|
||||
utilizationPercent: 5,
|
||||
},
|
||||
performance: {
|
||||
cacheHitRatio: '96%',
|
||||
deadlocks: 0,
|
||||
slowQueries: 0,
|
||||
},
|
||||
databaseSize: '1 GB',
|
||||
top10Tables: [{ table_stats: [] }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { type Redis } from 'ioredis';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
describe('RedisHealthIndicator', () => {
|
||||
let service: RedisHealthIndicator;
|
||||
let mockRedis: jest.Mocked<
|
||||
Pick<Redis, 'ping' | 'info' | 'dbsize' | 'memory'>
|
||||
>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
|
||||
beforeEach(async () => {
|
||||
mockRedis = {
|
||||
ping: jest.fn(),
|
||||
info: jest.fn(),
|
||||
dbsize: jest.fn(),
|
||||
memory: jest.fn(),
|
||||
};
|
||||
|
||||
const mockRedisService = {
|
||||
getClient: () => mockRedis,
|
||||
} as unknown as RedisClientService;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockReturnValue({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
redis: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((error) => ({
|
||||
redis: {
|
||||
status: 'down',
|
||||
...error,
|
||||
},
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
RedisHealthIndicator,
|
||||
{
|
||||
provide: RedisClientService,
|
||||
useValue: mockRedisService,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<RedisHealthIndicator>(RedisHealthIndicator);
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return up status with details when redis responds', async () => {
|
||||
// ai generated mock
|
||||
mockRedis.info
|
||||
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
|
||||
)
|
||||
.mockResolvedValueOnce('connected_clients:5\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.redis.status).toBe('up');
|
||||
expect(result.redis.details).toBeDefined();
|
||||
expect(result.redis.details.system.version).toBe('7.0.0');
|
||||
expect(result.redis.details.system.timestamp).toBeDefined();
|
||||
expect(result.redis.details.memory).toEqual({
|
||||
used: '1.2G',
|
||||
peak: '1.5G',
|
||||
fragmentation: 1.5,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return down status when redis fails', async () => {
|
||||
mockRedis.info.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.redis.status).toBe('down');
|
||||
expect(result.redis.message).toBe(
|
||||
HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
|
||||
);
|
||||
expect(result.redis.details.system.timestamp).toBeDefined();
|
||||
expect(result.redis.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should timeout after specified duration', async () => {
|
||||
mockRedis.info.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
const healthCheckPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
|
||||
const result = await healthCheckPromise;
|
||||
|
||||
expect(result.redis.status).toBe('down');
|
||||
expect(result.redis.message).toBe(HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT);
|
||||
expect(result.redis.details.system.timestamp).toBeDefined();
|
||||
expect(result.redis.details.stateHistory).toBeDefined();
|
||||
});
|
||||
|
||||
it('should maintain state history across health checks', async () => {
|
||||
// First check - healthy state
|
||||
mockRedis.info
|
||||
.mockResolvedValueOnce('redis_version:7.0.0\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'used_memory_human:1.2G\r\nused_memory_peak_human:1.5G\r\nmem_fragmentation_ratio:1.5\r\n',
|
||||
)
|
||||
.mockResolvedValueOnce('connected_clients:5\r\n')
|
||||
.mockResolvedValueOnce(
|
||||
'total_connections_received:100\r\nkeyspace_hits:90\r\nkeyspace_misses:10\r\n',
|
||||
);
|
||||
|
||||
await service.isHealthy();
|
||||
|
||||
// Second check - error state
|
||||
mockRedis.info.mockRejectedValueOnce(
|
||||
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.redis.details.stateHistory).toBeDefined();
|
||||
expect(result.redis.details.stateHistory.age).toBeDefined();
|
||||
expect(result.redis.details.stateHistory.timestamp).toBeDefined();
|
||||
expect(result.redis.details.stateHistory.details).toBeDefined();
|
||||
});
|
||||
});
|
||||
+348
@@ -0,0 +1,348 @@
|
||||
import { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { type Redis } from 'ioredis';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
const mockQueueInstance = {
|
||||
getWorkers: jest.fn().mockResolvedValue([]),
|
||||
close: jest.fn().mockResolvedValue(undefined),
|
||||
getMetrics: jest.fn().mockResolvedValue({ count: 0, data: [] }),
|
||||
getWaitingCount: jest.fn().mockResolvedValue(0),
|
||||
getActiveCount: jest.fn().mockResolvedValue(0),
|
||||
getDelayedCount: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
|
||||
jest.mock('bullmq', () => ({
|
||||
Queue: jest.fn(() => mockQueueInstance),
|
||||
}));
|
||||
|
||||
describe('WorkerHealthIndicator', () => {
|
||||
let service: WorkerHealthIndicator;
|
||||
let mockRedis: jest.Mocked<Pick<Redis, 'ping'>>;
|
||||
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
|
||||
let loggerSpy: jest.SpyInstance;
|
||||
|
||||
beforeEach(async () => {
|
||||
mockRedis = {
|
||||
ping: jest.fn(),
|
||||
};
|
||||
|
||||
const mockRedisService = {
|
||||
getClient: () => mockRedis,
|
||||
getQueueClient: () => mockRedis,
|
||||
} as unknown as RedisClientService;
|
||||
|
||||
healthIndicatorService = {
|
||||
check: jest.fn().mockReturnValue({
|
||||
up: jest.fn().mockImplementation((data) => ({
|
||||
worker: { status: 'up', ...data },
|
||||
})),
|
||||
down: jest.fn().mockImplementation((error) => ({
|
||||
worker: { status: 'down', error },
|
||||
})),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
WorkerHealthIndicator,
|
||||
{
|
||||
provide: RedisClientService,
|
||||
useValue: mockRedisService,
|
||||
},
|
||||
{
|
||||
provide: HealthIndicatorService,
|
||||
useValue: healthIndicatorService,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<WorkerHealthIndicator>(WorkerHealthIndicator);
|
||||
|
||||
loggerSpy = jest
|
||||
.spyOn(service['logger'], 'error')
|
||||
.mockImplementation(() => {});
|
||||
jest.useFakeTimers();
|
||||
|
||||
// Reset mocks to their default success state before each test
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([]);
|
||||
mockQueueInstance.getMetrics.mockResolvedValue({ count: 0, data: [] });
|
||||
mockQueueInstance.getWaitingCount.mockResolvedValue(0);
|
||||
mockQueueInstance.getActiveCount.mockResolvedValue(0);
|
||||
mockQueueInstance.getDelayedCount.mockResolvedValue(0);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it('should return up status when workers are active', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('up');
|
||||
expect('queues' in result.worker).toBe(true);
|
||||
if ('queues' in result.worker) {
|
||||
expect(result.worker.queues.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should return down status when no workers are active', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([]);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('down');
|
||||
expect('error' in result.worker).toBe(true);
|
||||
if ('error' in result.worker) {
|
||||
expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS);
|
||||
}
|
||||
});
|
||||
|
||||
it('should timeout after specified duration', async () => {
|
||||
jest.useFakeTimers();
|
||||
mockQueueInstance.getWorkers.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
|
||||
),
|
||||
);
|
||||
|
||||
const resultPromise = service.isHealthy();
|
||||
|
||||
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 200);
|
||||
const result = await resultPromise;
|
||||
|
||||
expect(result.worker.status).toBe('down');
|
||||
expect('error' in result.worker).toBe(true);
|
||||
if ('error' in result.worker) {
|
||||
expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT);
|
||||
}
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should check all message queues', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
|
||||
await service.isHealthy();
|
||||
|
||||
expect(mockQueueInstance.getWorkers).toHaveBeenCalledTimes(
|
||||
Object.keys(MessageQueue).length,
|
||||
);
|
||||
expect(mockQueueInstance.close).toHaveBeenCalledTimes(
|
||||
Object.keys(MessageQueue).length,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return down status when failure rate exceeds threshold', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({ count: 600 });
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({ count: 400 });
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0 });
|
||||
});
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('up');
|
||||
expect('queues' in result.worker).toBe(true);
|
||||
if ('queues' in result.worker) {
|
||||
expect(result.worker.queues[0].status).toBe('down');
|
||||
expect(result.worker.queues[0].metrics).toEqual({
|
||||
failed: 600,
|
||||
completed: 400,
|
||||
waiting: 0,
|
||||
active: 0,
|
||||
delayed: 0,
|
||||
failureRate: 60,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('should return complete metrics for active workers', async () => {
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({ count: 10 });
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({ count: 90 });
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0 });
|
||||
});
|
||||
mockQueueInstance.getWaitingCount.mockResolvedValue(5);
|
||||
mockQueueInstance.getActiveCount.mockResolvedValue(2);
|
||||
mockQueueInstance.getDelayedCount.mockResolvedValue(1);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('up');
|
||||
expect('queues' in result.worker).toBe(true);
|
||||
if ('queues' in result.worker) {
|
||||
expect(result.worker.queues[0].metrics).toEqual({
|
||||
failed: 10,
|
||||
completed: 90,
|
||||
waiting: 5,
|
||||
active: 2,
|
||||
delayed: 1,
|
||||
failureRate: 10,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle queue errors gracefully', async () => {
|
||||
mockQueueInstance.getWorkers.mockRejectedValue(new Error('Queue error'));
|
||||
mockQueueInstance.getMetrics.mockRejectedValue(new Error('Queue error'));
|
||||
mockQueueInstance.getWaitingCount.mockRejectedValue(
|
||||
new Error('Queue error'),
|
||||
);
|
||||
mockQueueInstance.getActiveCount.mockRejectedValue(
|
||||
new Error('Queue error'),
|
||||
);
|
||||
mockQueueInstance.getDelayedCount.mockRejectedValue(
|
||||
new Error('Queue error'),
|
||||
);
|
||||
|
||||
const result = await service.isHealthy();
|
||||
|
||||
expect(result.worker.status).toBe('down');
|
||||
expect('error' in result.worker).toBe(true);
|
||||
if ('error' in result.worker) {
|
||||
expect(result.worker.error).toBe(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS);
|
||||
}
|
||||
|
||||
expect(loggerSpy).toHaveBeenCalled();
|
||||
Object.values(MessageQueue).forEach((queueName) => {
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
`Error getting queue details for ${queueName}: Queue error`,
|
||||
);
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
`Error checking worker for queue ${queueName}: Queue error`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getQueueDetails', () => {
|
||||
beforeEach(() => {
|
||||
// Reset mocks to clean state before each test in this describe block
|
||||
mockQueueInstance.getWorkers.mockResolvedValue([{ id: 'worker1' }]);
|
||||
mockQueueInstance.getMetrics.mockResolvedValue({ count: 0, data: [] });
|
||||
});
|
||||
|
||||
it('should return metrics with time series data when pointsNeeded is provided', async () => {
|
||||
const pointsNeeded = 60;
|
||||
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({
|
||||
count: 10,
|
||||
data: Array(pointsNeeded).fill(10 / pointsNeeded),
|
||||
});
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({
|
||||
count: 90,
|
||||
data: Array(pointsNeeded).fill(90 / pointsNeeded),
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0, data: [] });
|
||||
});
|
||||
|
||||
const result = await service.getQueueDetails(
|
||||
MessageQueue.messagingQueue,
|
||||
{
|
||||
pointsNeeded,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.metrics).toMatchObject({
|
||||
failed: 10,
|
||||
completed: 90,
|
||||
failedData: expect.any(Array),
|
||||
completedData: expect.any(Array),
|
||||
});
|
||||
expect(result?.metrics.failedData).toHaveLength(pointsNeeded);
|
||||
expect(result?.metrics.completedData).toHaveLength(pointsNeeded);
|
||||
});
|
||||
|
||||
it('should handle invalid metrics data gracefully', async () => {
|
||||
const invalidData = ['invalid', null, undefined, '1', 2];
|
||||
|
||||
mockQueueInstance.getMetrics.mockResolvedValue({
|
||||
count: 0,
|
||||
data: invalidData,
|
||||
});
|
||||
|
||||
const result = await service.getQueueDetails(
|
||||
MessageQueue.messagingQueue,
|
||||
{
|
||||
pointsNeeded: 5,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.metrics.failedData).toEqual([NaN, 0, NaN, 1, 2]);
|
||||
expect(result?.metrics.completedData).toEqual([NaN, 0, NaN, 1, 2]);
|
||||
});
|
||||
|
||||
it('should calculate correct failure rate with time series data', async () => {
|
||||
mockQueueInstance.getMetrics.mockImplementation((type) => {
|
||||
if (type === 'failed') {
|
||||
return Promise.resolve({ count: 600, data: Array(10).fill(60) });
|
||||
}
|
||||
if (type === 'completed') {
|
||||
return Promise.resolve({ count: 400, data: Array(10).fill(40) });
|
||||
}
|
||||
|
||||
return Promise.resolve({ count: 0, data: [] });
|
||||
});
|
||||
|
||||
const result = await service.getQueueDetails(
|
||||
MessageQueue.messagingQueue,
|
||||
{
|
||||
pointsNeeded: 10,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.metrics).toMatchObject({
|
||||
failed: 600,
|
||||
completed: 400,
|
||||
failureRate: 60,
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle queue errors gracefully', async () => {
|
||||
mockQueueInstance.getWorkers.mockRejectedValue(new Error('Queue error'));
|
||||
mockQueueInstance.getMetrics.mockRejectedValue(new Error('Queue error'));
|
||||
|
||||
await expect(
|
||||
service.getQueueDetails(MessageQueue.messagingQueue),
|
||||
).rejects.toThrow('Queue error');
|
||||
|
||||
expect(loggerSpy).toHaveBeenCalledWith(
|
||||
`Error getting queue details for ${MessageQueue.messagingQueue}: Queue error`,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
+7
-7
@@ -12,13 +12,13 @@ import { type QueueMetricsDataDTO } from 'src/engine/core-modules/admin-panel/dt
|
||||
import { type SystemHealthDTO } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/health/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
|
||||
import { type WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-queue-health.type';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health';
|
||||
import { type WorkerQueueHealth } from 'src/engine/core-modules/admin-panel/types/worker-queue-health.type';
|
||||
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
|
||||
@@ -12,22 +12,28 @@ import { AuthModule } from 'src/engine/core-modules/auth/auth.module';
|
||||
import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
|
||||
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
||||
import { FileModule } from 'src/engine/core-modules/file/file.module';
|
||||
import { HealthModule } from 'src/engine/core-modules/health/health.module';
|
||||
import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health';
|
||||
import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health';
|
||||
import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health';
|
||||
import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health';
|
||||
import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health';
|
||||
import { ImpersonationModule } from 'src/engine/core-modules/impersonation/impersonation.module';
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
|
||||
import { TelemetryModule } from 'src/engine/core-modules/telemetry/telemetry.module';
|
||||
import { UserEntity } from 'src/engine/core-modules/user/user.entity';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([UserEntity]),
|
||||
TypeOrmModule.forFeature([UserEntity, WorkspaceEntity]),
|
||||
AuthModule,
|
||||
FileModule,
|
||||
WorkspaceDomainsModule,
|
||||
HealthModule,
|
||||
RedisClientModule,
|
||||
TerminusModule,
|
||||
MetricsModule,
|
||||
FeatureFlagModule,
|
||||
AuditModule,
|
||||
TelemetryModule,
|
||||
@@ -40,6 +46,11 @@ import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permi
|
||||
AdminPanelHealthService,
|
||||
AdminPanelQueueService,
|
||||
SecureHttpClientService,
|
||||
DatabaseHealthIndicator,
|
||||
RedisHealthIndicator,
|
||||
WorkerHealthIndicator,
|
||||
ConnectedAccountHealth,
|
||||
AppHealthIndicator,
|
||||
],
|
||||
exports: [AdminPanelService],
|
||||
})
|
||||
|
||||
@@ -25,7 +25,7 @@ import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/service
|
||||
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
|
||||
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
|
||||
import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
|
||||
import { ConfigVariableGraphqlApiExceptionFilter } from 'src/engine/core-modules/twenty-config/filters/config-variable-graphql-api-exception.filter';
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
export const HEALTH_ERROR_MESSAGES = {
|
||||
NO_ACTIVE_WORKERS: 'No active workers found',
|
||||
WORKER_TIMEOUT: 'Worker check timeout',
|
||||
DATABASE_TIMEOUT: 'Database timeout',
|
||||
REDIS_TIMEOUT: 'Redis timeout',
|
||||
DATABASE_CONNECTION_FAILED: 'Database connection failed',
|
||||
REDIS_CONNECTION_FAILED: 'Unknown Redis error',
|
||||
WORKER_CHECK_FAILED: 'Worker check failed',
|
||||
MESSAGE_SYNC_TIMEOUT: 'Message sync check timeout',
|
||||
MESSAGE_SYNC_CHECK_FAILED: 'Message sync check failed',
|
||||
MESSAGE_SYNC_HIGH_FAILURE_RATE: 'High failure rate in message sync jobs',
|
||||
CALENDAR_SYNC_TIMEOUT: 'Calendar sync check timeout',
|
||||
CALENDAR_SYNC_CHECK_FAILED: 'Calendar sync check failed',
|
||||
CALENDAR_SYNC_HIGH_FAILURE_RATE: 'High failure rate in calendar sync jobs',
|
||||
APP_HEALTH_CHECK_FAILED: 'App health check failed',
|
||||
} as const;
|
||||
+1
@@ -0,0 +1 @@
|
||||
export const HEALTH_INDICATORS_TIMEOUT = 3000;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
|
||||
type HealthIndicatorInfo = {
|
||||
id: HealthIndicatorId;
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
export const METRICS_FAILURE_RATE_THRESHOLD = 20;
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import { Field } from '@nestjs/graphql';
|
||||
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
|
||||
export class HealthIndicatorInput {
|
||||
@Field(() => HealthIndicatorId)
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { AdminPanelWorkerQueueHealthDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto';
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
|
||||
@ObjectType('AdminPanelHealthServiceData')
|
||||
export class AdminPanelHealthServiceDataDTO {
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { QueueMetricsSeriesDTO } from 'src/engine/core-modules/admin-panel/dtos/queue-metrics-series.dto';
|
||||
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
|
||||
import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-queue-metrics.type';
|
||||
import { WorkerQueueMetrics } from 'src/engine/core-modules/admin-panel/types/worker-queue-metrics.type';
|
||||
|
||||
@ObjectType('QueueMetricsData')
|
||||
export class QueueMetricsDataDTO {
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
|
||||
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
|
||||
|
||||
@ObjectType('SystemHealthService')
|
||||
export class SystemHealthServiceDTO {
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
import { registerEnumType } from '@nestjs/graphql';
|
||||
|
||||
export enum HealthIndicatorId {
|
||||
database = 'database',
|
||||
redis = 'redis',
|
||||
worker = 'worker',
|
||||
connectedAccount = 'connectedAccount',
|
||||
app = 'app',
|
||||
}
|
||||
|
||||
registerEnumType(HealthIndicatorId, {
|
||||
name: 'HealthIndicatorId',
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
type HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { HealthStateManager } from 'src/engine/core-modules/admin-panel/utils/health-state-manager.util';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
|
||||
@Injectable()
|
||||
export class AppHealthIndicator {
|
||||
private stateManager = new HealthStateManager();
|
||||
|
||||
constructor(
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
@InjectRepository(WorkspaceEntity)
|
||||
private readonly workspaceRepository: Repository<WorkspaceEntity>,
|
||||
) {}
|
||||
|
||||
// TODO refactor, a workspace health should be based on its app versioning
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('app');
|
||||
|
||||
try {
|
||||
const totalWorkspaceCount = await this.workspaceRepository.count();
|
||||
|
||||
const details = {
|
||||
system: {
|
||||
nodeVersion: process.version,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
overview: {
|
||||
totalWorkspacesCount: totalWorkspaceCount,
|
||||
erroredWorkspaceCount: 0,
|
||||
},
|
||||
erroredWorkspace: 0,
|
||||
};
|
||||
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.up({ details });
|
||||
} catch (error) {
|
||||
const stateWithAge = this.stateManager.getStateWithAge();
|
||||
|
||||
return indicator.down({
|
||||
message: error.message,
|
||||
details: {
|
||||
system: {
|
||||
nodeVersion: process.version,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
stateHistory: stateWithAge,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
type HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util';
|
||||
import {
|
||||
CALENDAR_SYNC_METRICS_BY_STATUS,
|
||||
MESSAGE_SYNC_METRICS_BY_STATUS,
|
||||
} from 'src/engine/core-modules/metrics/constants/account-sync-metrics-by-status.constant';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
@Injectable()
|
||||
export class ConnectedAccountHealth {
|
||||
constructor(
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
private readonly metricsService: MetricsService,
|
||||
) {}
|
||||
|
||||
private async checkMessageSyncHealth(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('messageSync');
|
||||
|
||||
try {
|
||||
const counters = await withHealthCheckTimeout(
|
||||
this.metricsService.groupMetrics(MESSAGE_SYNC_METRICS_BY_STATUS),
|
||||
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
|
||||
);
|
||||
|
||||
const totalJobs = Object.values(counters).reduce(
|
||||
(sum, count) => sum + (count || 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const failedJobs = counters.FAILED_UNKNOWN || 0;
|
||||
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
|
||||
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
|
||||
: 0;
|
||||
const details = {
|
||||
counters,
|
||||
totalJobs,
|
||||
failedJobs,
|
||||
failureRate,
|
||||
};
|
||||
|
||||
if (totalJobs === 0 || failureRate < METRICS_FAILURE_RATE_THRESHOLD) {
|
||||
return indicator.up({ details });
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
|
||||
details,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error.message === HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED;
|
||||
|
||||
return indicator.down({
|
||||
error: errorMessage,
|
||||
details: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async checkCalendarSyncHealth(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('calendarSync');
|
||||
|
||||
try {
|
||||
const counters = await withHealthCheckTimeout(
|
||||
this.metricsService.groupMetrics(CALENDAR_SYNC_METRICS_BY_STATUS),
|
||||
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
|
||||
);
|
||||
|
||||
const totalJobs = Object.values(counters).reduce(
|
||||
(sum, count) => sum + (count || 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const failedJobs = counters.FAILED_UNKNOWN || 0;
|
||||
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
|
||||
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
|
||||
: 0;
|
||||
const details = {
|
||||
counters,
|
||||
totalJobs,
|
||||
failedJobs,
|
||||
failureRate,
|
||||
};
|
||||
|
||||
if (totalJobs === 0 || failureRate < METRICS_FAILURE_RATE_THRESHOLD) {
|
||||
return indicator.up({ details });
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error: HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
|
||||
details,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error.message === HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_CHECK_FAILED;
|
||||
|
||||
return indicator.down({
|
||||
error: errorMessage,
|
||||
details: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('connectedAccount');
|
||||
|
||||
const [messageResult, calendarResult] = await Promise.all([
|
||||
this.checkMessageSyncHealth(),
|
||||
this.checkCalendarSyncHealth(),
|
||||
]);
|
||||
|
||||
const isMessageSyncDown = messageResult.messageSync.status === 'down';
|
||||
const isCalendarSyncDown = calendarResult.calendarSync.status === 'down';
|
||||
|
||||
if (isMessageSyncDown || isCalendarSyncDown) {
|
||||
let error: string;
|
||||
|
||||
if (isMessageSyncDown && isCalendarSyncDown) {
|
||||
error = `${messageResult.messageSync.error} and ${calendarResult.calendarSync.error}`;
|
||||
} else if (isMessageSyncDown) {
|
||||
error = messageResult.messageSync.error;
|
||||
} else {
|
||||
error = calendarResult.calendarSync.error;
|
||||
}
|
||||
|
||||
return indicator.down({
|
||||
error,
|
||||
details: {
|
||||
messageSync: messageResult.messageSync,
|
||||
calendarSync: calendarResult.calendarSync,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return indicator.up({
|
||||
details: {
|
||||
messageSync: messageResult.messageSync,
|
||||
calendarSync: calendarResult.calendarSync,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
type HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util';
|
||||
import { HealthStateManager } from 'src/engine/core-modules/admin-panel/utils/health-state-manager.util';
|
||||
|
||||
@Injectable()
|
||||
export class DatabaseHealthIndicator {
|
||||
private stateManager = new HealthStateManager();
|
||||
|
||||
constructor(
|
||||
@InjectDataSource()
|
||||
private readonly dataSource: DataSource,
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
) {}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('database');
|
||||
|
||||
try {
|
||||
const [
|
||||
[versionResult],
|
||||
[activeConnections],
|
||||
[maxConnections],
|
||||
[uptime],
|
||||
[databaseSize],
|
||||
tableStats,
|
||||
[cacheHitRatio],
|
||||
[deadlocks],
|
||||
[slowQueries],
|
||||
] = await withHealthCheckTimeout(
|
||||
Promise.all([
|
||||
this.dataSource.query('SELECT version()'),
|
||||
this.dataSource.query(
|
||||
'SELECT count(*) as count FROM pg_stat_activity',
|
||||
),
|
||||
this.dataSource.query('SHOW max_connections'),
|
||||
this.dataSource.query(
|
||||
'SELECT extract(epoch from current_timestamp - pg_postmaster_start_time()) as uptime',
|
||||
),
|
||||
this.dataSource.query(
|
||||
'SELECT pg_size_pretty(pg_database_size(current_database())) as size',
|
||||
),
|
||||
this.dataSource.query(`
|
||||
SELECT schemaname, relname, n_live_tup, n_dead_tup, last_vacuum, last_autovacuum
|
||||
FROM pg_stat_user_tables
|
||||
ORDER BY n_live_tup DESC
|
||||
LIMIT 10
|
||||
`),
|
||||
this.dataSource.query(`
|
||||
SELECT
|
||||
sum(heap_blks_hit) * 100.0 / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio
|
||||
FROM pg_statio_user_tables
|
||||
`),
|
||||
this.dataSource.query(
|
||||
'SELECT deadlocks FROM pg_stat_database WHERE datname = current_database()',
|
||||
),
|
||||
this.dataSource.query(`
|
||||
SELECT count(*) as count
|
||||
FROM pg_stat_activity
|
||||
WHERE state = 'active'
|
||||
AND query_start < now() - interval '1 minute'
|
||||
`),
|
||||
]),
|
||||
HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT,
|
||||
);
|
||||
|
||||
const details = {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
version: versionResult.version,
|
||||
uptime: Math.round(uptime.uptime / 3600) + ' hours',
|
||||
},
|
||||
connections: {
|
||||
active: parseInt(activeConnections.count),
|
||||
max: parseInt(maxConnections.max_connections),
|
||||
utilizationPercent: Math.round(
|
||||
(parseInt(activeConnections.count) /
|
||||
parseInt(maxConnections.max_connections)) *
|
||||
100,
|
||||
),
|
||||
},
|
||||
databaseSize: databaseSize.size,
|
||||
performance: {
|
||||
cacheHitRatio: Math.round(parseFloat(cacheHitRatio.ratio)) + '%',
|
||||
deadlocks: parseInt(deadlocks.deadlocks),
|
||||
slowQueries: parseInt(slowQueries.count),
|
||||
},
|
||||
top10Tables: tableStats,
|
||||
};
|
||||
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.up({ details });
|
||||
} catch (error) {
|
||||
const message =
|
||||
error.message === HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.DATABASE_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED;
|
||||
|
||||
const stateWithAge = this.stateManager.getStateWithAge();
|
||||
|
||||
return indicator.down({
|
||||
message,
|
||||
details: {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
stateHistory: stateWithAge,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
type HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util';
|
||||
import { HealthStateManager } from 'src/engine/core-modules/admin-panel/utils/health-state-manager.util';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
@Injectable()
|
||||
export class RedisHealthIndicator {
|
||||
private stateManager = new HealthStateManager();
|
||||
|
||||
constructor(
|
||||
private readonly redisClient: RedisClientService,
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
) {}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('redis');
|
||||
|
||||
try {
|
||||
const [info, memory, clients, stats] = await withHealthCheckTimeout(
|
||||
Promise.all([
|
||||
this.redisClient.getClient().info(),
|
||||
this.redisClient.getClient().info('memory'),
|
||||
this.redisClient.getClient().info('clients'),
|
||||
this.redisClient.getClient().info('stats'),
|
||||
]),
|
||||
HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT,
|
||||
);
|
||||
|
||||
const parseInfo = (info: string) => {
|
||||
const result: Record<string, string> = {};
|
||||
|
||||
info.split('\r\n').forEach((line) => {
|
||||
const [key, value] = line.split(':');
|
||||
|
||||
if (key && value) {
|
||||
result[key] = value;
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const infoData = parseInfo(info);
|
||||
const memoryData = parseInfo(memory);
|
||||
const clientsData = parseInfo(clients);
|
||||
const statsData = parseInfo(stats);
|
||||
|
||||
const details = {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
version: infoData.redis_version,
|
||||
uptime:
|
||||
Math.round(parseInt(infoData.uptime_in_seconds) / 3600) + ' hours',
|
||||
},
|
||||
memory: {
|
||||
used: memoryData.used_memory_human,
|
||||
peak: memoryData.used_memory_peak_human,
|
||||
fragmentation: parseFloat(memoryData.mem_fragmentation_ratio),
|
||||
},
|
||||
connections: {
|
||||
current: parseInt(clientsData.connected_clients),
|
||||
total: parseInt(statsData.total_connections_received),
|
||||
rejected: parseInt(statsData.rejected_connections),
|
||||
},
|
||||
performance: {
|
||||
opsPerSecond: parseInt(statsData.instantaneous_ops_per_sec),
|
||||
hitRate: statsData.keyspace_hits
|
||||
? Math.round(
|
||||
(parseInt(statsData.keyspace_hits) /
|
||||
(parseInt(statsData.keyspace_hits) +
|
||||
parseInt(statsData.keyspace_misses))) *
|
||||
100,
|
||||
) + '%'
|
||||
: '0%',
|
||||
evictedKeys: parseInt(statsData.evicted_keys),
|
||||
expiredKeys: parseInt(statsData.expired_keys),
|
||||
},
|
||||
replication: {
|
||||
role: infoData.role,
|
||||
connectedSlaves: parseInt(infoData.connected_slaves || '0'),
|
||||
},
|
||||
};
|
||||
|
||||
this.stateManager.updateState(details);
|
||||
|
||||
return indicator.up({ details });
|
||||
} catch (error) {
|
||||
const message =
|
||||
error.message === HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.REDIS_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED;
|
||||
|
||||
const stateWithAge = this.stateManager.getStateWithAge();
|
||||
|
||||
return indicator.down({
|
||||
message,
|
||||
details: {
|
||||
system: {
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
stateHistory: stateWithAge,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import {
|
||||
type HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/admin-panel/constants/health-error-messages.constants';
|
||||
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/admin-panel/constants/metrics-failure-rate-threshold.const';
|
||||
import { type WorkerQueueHealth } from 'src/engine/core-modules/admin-panel/types/worker-queue-health.type';
|
||||
import { withHealthCheckTimeout } from 'src/engine/core-modules/admin-panel/utils/health-check-timeout.util';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
|
||||
|
||||
@Injectable()
|
||||
export class WorkerHealthIndicator {
|
||||
private readonly logger = new Logger(WorkerHealthIndicator.name);
|
||||
|
||||
constructor(
|
||||
private readonly redisClient: RedisClientService,
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
) {}
|
||||
|
||||
async isHealthy(): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check('worker');
|
||||
|
||||
try {
|
||||
const workerStatus = await withHealthCheckTimeout(
|
||||
this.checkWorkers(),
|
||||
HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT,
|
||||
);
|
||||
|
||||
if (workerStatus.status === 'up') {
|
||||
return indicator.up({
|
||||
queues: workerStatus.queues,
|
||||
});
|
||||
}
|
||||
|
||||
return indicator.down(workerStatus.error);
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error.message === HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT
|
||||
? HEALTH_ERROR_MESSAGES.WORKER_TIMEOUT
|
||||
: HEALTH_ERROR_MESSAGES.WORKER_CHECK_FAILED;
|
||||
|
||||
return indicator.down(errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
async getQueueDetails(
|
||||
queueName: MessageQueue,
|
||||
options?: {
|
||||
pointsNeeded?: number;
|
||||
},
|
||||
): Promise<WorkerQueueHealth | null> {
|
||||
const redis = this.redisClient.getQueueClient();
|
||||
const queue = new Queue(queueName, { connection: redis });
|
||||
|
||||
try {
|
||||
const workers = await queue.getWorkers();
|
||||
|
||||
if (workers.length > 0) {
|
||||
const metricsParams = options?.pointsNeeded
|
||||
? [0, options.pointsNeeded - 1]
|
||||
: [];
|
||||
|
||||
const [
|
||||
failedMetrics,
|
||||
completedMetrics,
|
||||
waitingCount,
|
||||
activeCount,
|
||||
delayedCount,
|
||||
] = await Promise.all([
|
||||
queue.getMetrics('failed', ...metricsParams),
|
||||
queue.getMetrics('completed', ...metricsParams),
|
||||
queue.getWaitingCount(),
|
||||
queue.getActiveCount(),
|
||||
queue.getDelayedCount(),
|
||||
]);
|
||||
|
||||
const failedCount = options?.pointsNeeded
|
||||
? this.calculateMetricsSum(failedMetrics.data)
|
||||
: failedMetrics.count;
|
||||
|
||||
const completedCount = options?.pointsNeeded
|
||||
? this.calculateMetricsSum(completedMetrics.data)
|
||||
: completedMetrics.count;
|
||||
|
||||
const totalJobs = failedCount + completedCount;
|
||||
const failureRate =
|
||||
totalJobs > 0
|
||||
? Number(((failedCount / totalJobs) * 100).toFixed(1))
|
||||
: 0;
|
||||
|
||||
return {
|
||||
queueName,
|
||||
workers: workers.length,
|
||||
status: failureRate > METRICS_FAILURE_RATE_THRESHOLD ? 'down' : 'up',
|
||||
metrics: {
|
||||
failed: failedCount,
|
||||
completed: completedCount,
|
||||
waiting: waitingCount,
|
||||
active: activeCount,
|
||||
delayed: delayedCount,
|
||||
failureRate,
|
||||
...(options?.pointsNeeded && {
|
||||
failedData: failedMetrics.data.map(Number),
|
||||
completedData: completedMetrics.data.map(Number),
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error getting queue details for ${queueName}: ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
await queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
private calculateMetricsSum(data: string[] | number[]): number {
|
||||
const sum = data.reduce((sum: number, value: string | number) => {
|
||||
const numericValue = Number(value);
|
||||
|
||||
return sum + (isNaN(numericValue) ? 0 : numericValue);
|
||||
}, 0);
|
||||
|
||||
return Math.round(Number(sum));
|
||||
}
|
||||
|
||||
private async checkWorkers() {
|
||||
const queues = Object.values(MessageQueue);
|
||||
const queueStatuses: WorkerQueueHealth[] = [];
|
||||
|
||||
for (const queueName of queues) {
|
||||
try {
|
||||
const queueDetails = await this.getQueueDetails(queueName);
|
||||
|
||||
if (queueDetails) {
|
||||
queueStatuses.push(queueDetails);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error checking worker for queue ${queueName}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const hasActiveWorkers = queueStatuses.some((q) => q.workers > 0);
|
||||
|
||||
return {
|
||||
status: hasActiveWorkers ? 'up' : 'down',
|
||||
error: hasActiveWorkers
|
||||
? undefined
|
||||
: HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS,
|
||||
queues: queueStatuses,
|
||||
};
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
@ObjectType()
|
||||
export class AccountSyncJobByStatusCounter {
|
||||
@Field(() => Number, { nullable: true })
|
||||
NOT_SYNCED?: number;
|
||||
|
||||
@Field(() => Number, { nullable: true })
|
||||
ACTIVE?: number;
|
||||
|
||||
@Field(() => Number, { nullable: true })
|
||||
FAILED_INSUFFICIENT_PERMISSIONS?: number;
|
||||
|
||||
@Field(() => Number, { nullable: true })
|
||||
FAILED_UNKNOWN?: number;
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { WorkerQueueMetrics } from 'src/engine/core-modules/admin-panel/types/worker-queue-metrics.type';
|
||||
|
||||
@ObjectType()
|
||||
export class WorkerQueueHealth {
|
||||
@Field(() => String)
|
||||
queueName: string;
|
||||
|
||||
@Field(() => String)
|
||||
status: string;
|
||||
|
||||
@Field(() => Number)
|
||||
workers: number;
|
||||
|
||||
@Field(() => WorkerQueueMetrics)
|
||||
metrics: WorkerQueueMetrics;
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
@ObjectType()
|
||||
export class WorkerQueueMetrics {
|
||||
@Field(() => Number)
|
||||
failed: number;
|
||||
|
||||
@Field(() => Number)
|
||||
completed: number;
|
||||
|
||||
@Field(() => Number)
|
||||
waiting: number;
|
||||
|
||||
@Field(() => Number)
|
||||
active: number;
|
||||
|
||||
@Field(() => Number)
|
||||
delayed: number;
|
||||
|
||||
@Field(() => Number)
|
||||
failureRate: number;
|
||||
|
||||
@Field(() => [Number], { nullable: true })
|
||||
failedData?: number[];
|
||||
|
||||
@Field(() => [Number], { nullable: true })
|
||||
completedData?: number[];
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts';
|
||||
|
||||
export const withHealthCheckTimeout = async <T>(
|
||||
promise: Promise<T>,
|
||||
errorMessage: string,
|
||||
): Promise<T> => {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise<T>((_, reject) =>
|
||||
setTimeout(
|
||||
() => reject(new Error(errorMessage)),
|
||||
HEALTH_INDICATORS_TIMEOUT,
|
||||
),
|
||||
),
|
||||
]);
|
||||
};
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
export class HealthStateManager {
|
||||
private lastKnownState: {
|
||||
timestamp: Date;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
details: Record<string, any>;
|
||||
} | null = null;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
updateState(details: Record<string, any>) {
|
||||
this.lastKnownState = {
|
||||
timestamp: new Date(),
|
||||
details,
|
||||
};
|
||||
}
|
||||
|
||||
getStateWithAge() {
|
||||
return this.lastKnownState
|
||||
? {
|
||||
...this.lastKnownState,
|
||||
age: Date.now() - this.lastKnownState.timestamp.getTime(),
|
||||
}
|
||||
: 'No previous state available';
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user