1635 extensibilitytwenty cli app vars (#15143)

- Update twenty-cli to support application env variable definition
- Update twenty-server to create a new `core.applicationVariable` entity
to store env variables and provide env var when executing serverless
function
- Update twenty-front to support application environment variable value
setting

<img width="1044" height="660" alt="image"
src="https://github.com/user-attachments/assets/24c3d323-5370-4a80-8174-fc4653cc3c22"
/>

<img width="1178" height="662" alt="image"
src="https://github.com/user-attachments/assets/c124f423-8ed8-4246-ae5b-a9bd6672c7dc"
/>

<img width="1163" height="823" alt="image"
src="https://github.com/user-attachments/assets/fb7425a3-facc-4895-a5eb-8a8e278e0951"
/>

<img width="1087" height="696" alt="image"
src="https://github.com/user-attachments/assets/113da8a2-5590-433c-b1b3-5ed3137f24ca"
/>

<img width="1512" height="715" alt="image"
src="https://github.com/user-attachments/assets/1d2110b7-301d-4f21-a45c-ddd54d6e3391"
/>

<img width="1287" height="581" alt="image"
src="https://github.com/user-attachments/assets/353b16c6-0527-444c-87d6-51447a96cbc7"
/>
This commit is contained in:
martmull
2025-10-17 10:54:38 +02:00
committed by GitHub
parent 54baa47fbb
commit d2e7f2a910
58 changed files with 1305 additions and 359 deletions
@@ -0,0 +1,23 @@
import { type MigrationInterface, type QueryRunner } from 'typeorm';
export class AddApplicationVariableCoreEntity1760640844181
implements MigrationInterface
{
name = 'AddApplicationVariableCoreEntity1760640844181';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "core"."applicationVariable" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "key" text NOT NULL, "value" text NOT NULL DEFAULT '', "description" text NOT NULL DEFAULT '', "isSecret" boolean NOT NULL DEFAULT false, "applicationId" uuid, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "IDX_APPLICATION_VARIABLE_KEY_APPLICATION_ID_UNIQUE" UNIQUE ("key", "applicationId"), CONSTRAINT "PK_62f7823eb5f1e416c9d60614dfb" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`ALTER TABLE "core"."applicationVariable" ADD CONSTRAINT "FK_51adb49e7f8df35dd23e01c4830" FOREIGN KEY ("applicationId") REFERENCES "core"."application"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."applicationVariable" DROP CONSTRAINT "FK_51adb49e7f8df35dd23e01c4830"`,
);
await queryRunner.query(`DROP TABLE "core"."applicationVariable"`);
}
}
@@ -28,6 +28,7 @@ import { FlatRouteTrigger } from 'src/engine/metadata-modules/route-trigger/type
import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service';
import { ServerlessFunctionV2Service } from 'src/engine/metadata-modules/serverless-function/services/serverless-function-v2.service';
import { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { ApplicationVariableService } from 'src/engine/core-modules/applicationVariable/application-variable.service';
@Injectable()
export class ApplicationSyncService {
@@ -35,6 +36,7 @@ export class ApplicationSyncService {
constructor(
private readonly applicationService: ApplicationService,
private readonly applicationVariableService: ApplicationVariableService,
private readonly serverlessFunctionLayerService: ServerlessFunctionLayerService,
private readonly objectMetadataServiceV2: ObjectMetadataServiceV2,
private readonly serverlessFunctionV2Service: ServerlessFunctionV2Service,
@@ -106,7 +108,7 @@ export class ApplicationSyncService {
workspaceId,
);
return await this.applicationService.create({
const application = await this.applicationService.create({
universalIdentifier: manifest.universalIdentifier,
name: manifest.name,
description: manifest.description,
@@ -115,6 +117,13 @@ export class ApplicationSyncService {
serverlessFunctionLayerId: serverlessFunctionLayer.id,
workspaceId,
});
await this.applicationVariableService.upsertManyApplicationVariables({
env: manifest.env,
applicationId: application.id,
});
return application;
}
await this.serverlessFunctionLayerService.update(
@@ -131,6 +140,11 @@ export class ApplicationSyncService {
version: manifest.version,
});
await this.applicationVariableService.upsertManyApplicationVariables({
env: manifest.env,
applicationId: application.id,
});
return application;
}
@@ -18,6 +18,7 @@ import { AgentEntity } from 'src/engine/metadata-modules/agent/agent.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ServerlessFunctionLayerEntity } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.entity';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ApplicationVariable } from 'src/engine/core-modules/applicationVariable/application-variable.entity';
@Entity({ name: 'application', schema: 'core' })
@Index('IDX_APPLICATION_WORKSPACE_ID', ['workspaceId'])
@@ -86,6 +87,15 @@ export class ApplicationEntity {
})
objects: Relation<ObjectMetadataEntity[]>;
@OneToMany(
() => ApplicationVariable,
(applicationVariable) => applicationVariable.application,
{
onDelete: 'CASCADE',
},
)
applicationVariables: Relation<ApplicationVariable[]>;
@ManyToOne(() => Workspace, {
onDelete: 'CASCADE',
})
@@ -16,6 +16,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
import { RouteTriggerModule } from 'src/engine/metadata-modules/route-trigger/route-trigger.module';
import { ServerlessFunctionLayerModule } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.module';
import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module';
import { ApplicationVariableModule } from 'src/engine/core-modules/applicationVariable/application-variable.module';
@Module({
imports: [
@@ -24,6 +25,7 @@ import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless
ObjectMetadataModule,
DataSourceModule,
AgentModule,
ApplicationVariableModule,
ServerlessFunctionLayerModule,
ServerlessFunctionModule,
DatabaseEventTriggerModule,
@@ -25,7 +25,12 @@ export class ApplicationService {
): Promise<ApplicationEntity[]> {
return this.applicationRepository.find({
where: { workspaceId },
relations: ['serverlessFunctions', 'agents', 'objects'],
relations: [
'serverlessFunctions',
'agents',
'objects',
'applicationVariables',
],
});
}
@@ -35,7 +40,12 @@ export class ApplicationService {
): Promise<ApplicationEntity> {
const application = await this.applicationRepository.findOne({
where: { workspaceId, id: applicationId },
relations: ['serverlessFunctions', 'agents', 'objects'],
relations: [
'serverlessFunctions',
'agents',
'objects',
'applicationVariables',
],
});
if (!isDefined(application)) {
@@ -6,6 +6,7 @@ import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/
import { ServerlessFunctionDTO } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { AgentDTO } from 'src/engine/metadata-modules/agent/dtos/agent.dto';
import { ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto';
import { ApplicationVariableDTO } from 'src/engine/core-modules/applicationVariable/dtos/application-variable.dto';
@ObjectType('Application')
export class ApplicationDTO {
@@ -22,6 +23,10 @@ export class ApplicationDTO {
@Field()
description: string;
@IsString()
@Field()
version: string;
@Field(() => [AgentDTO])
agents: AgentDTO[];
@@ -30,4 +35,7 @@ export class ApplicationDTO {
@Field(() => [ObjectMetadataDTO])
objects: ObjectMetadataDTO[];
@Field(() => [ApplicationVariableDTO])
applicationVariables: ApplicationVariableDTO[];
}
@@ -11,6 +11,15 @@ export type PackageJson = {
npm: string;
yarn: string;
};
env: Record<
string,
{
key: string;
value?: string;
description?: string;
isSecret: boolean;
}
>;
icon?: string;
version: string;
dependencies?: object;
@@ -0,0 +1,21 @@
import { Catch, ExceptionFilter } from '@nestjs/common';
import { assertUnreachable } from 'twenty-shared/utils';
import {
ApplicationVariableException,
ApplicationVariableExceptionCode,
} from 'src/engine/core-modules/applicationVariable/application-variable.exception';
import { NotFoundError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
@Catch(ApplicationVariableException)
export class ApplicationVariableExceptionFilter implements ExceptionFilter {
catch(exception: ApplicationVariableException) {
switch (exception.code) {
case ApplicationVariableExceptionCode.APPLICATION_VARIABLE_NOT_FOUND:
throw new NotFoundError(exception);
default:
assertUnreachable(exception.code);
}
}
}
@@ -0,0 +1,65 @@
import { ObjectType } from '@nestjs/graphql';
import {
Column,
CreateDateColumn,
Entity,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
Unique,
UpdateDateColumn,
} from 'typeorm';
import { IDField } from '@ptc-org/nestjs-query-graphql';
import { Relation } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/relation.interface';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
@Entity({
name: 'applicationVariable',
schema: 'core',
})
@ObjectType()
@Unique('IDX_APPLICATION_VARIABLE_KEY_APPLICATION_ID_UNIQUE', [
'key',
'applicationId',
])
export class ApplicationVariable {
@IDField(() => UUIDScalarType)
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false, type: 'text' })
key: string;
@Column({ nullable: false, type: 'text', default: '' })
value: string;
@Column({ nullable: false, type: 'text', default: '' })
description: string;
@Column({ nullable: false, type: 'boolean', default: false })
isSecret: boolean;
@Column({ nullable: true, type: 'uuid' })
applicationId?: string;
@ManyToOne(
() => ApplicationEntity,
(application) => application.applicationVariables,
{
onDelete: 'CASCADE',
nullable: true,
},
)
@JoinColumn({ name: 'applicationId' })
application: Relation<ApplicationEntity> | null;
@CreateDateColumn({ type: 'timestamptz' })
createdAt: Date;
@UpdateDateColumn({ type: 'timestamptz' })
updatedAt: Date;
}
@@ -0,0 +1,7 @@
import { CustomException } from 'src/utils/custom-exception';
export class ApplicationVariableException extends CustomException<ApplicationVariableExceptionCode> {}
export enum ApplicationVariableExceptionCode {
APPLICATION_VARIABLE_NOT_FOUND = 'APPLICATION_VARIABLE_NOT_FOUND',
}
@@ -0,0 +1,14 @@
import { Module } from '@nestjs/common';
import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
import { ApplicationVariable } from 'src/engine/core-modules/applicationVariable/application-variable.entity';
import { ApplicationVariableService } from 'src/engine/core-modules/applicationVariable/application-variable.service';
import { ApplicationVariableResolver } from 'src/engine/core-modules/applicationVariable/application-variable.resolver';
@Module({
imports: [NestjsQueryTypeOrmModule.forFeature([ApplicationVariable])],
providers: [ApplicationVariableService, ApplicationVariableResolver],
exports: [ApplicationVariableService],
})
export class ApplicationVariableModule {}
@@ -0,0 +1,25 @@
import { UseFilters, UseGuards } from '@nestjs/common';
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { ApplicationVariableService } from 'src/engine/core-modules/applicationVariable/application-variable.service';
import { UpdateApplicationVariableInput } from 'src/engine/core-modules/applicationVariable/dtos/update-application-variable.input';
import { ApplicationVariableExceptionFilter } from 'src/engine/core-modules/applicationVariable/application-variable-exception-filter';
@UseGuards(WorkspaceAuthGuard)
@Resolver()
@UseFilters(ApplicationVariableExceptionFilter)
export class ApplicationVariableResolver {
constructor(
private readonly applicationVariableService: ApplicationVariableService,
) {}
@Mutation(() => Boolean)
async updateOneApplicationVariable(
@Args() { key, value, applicationId }: UpdateApplicationVariableInput,
) {
await this.applicationVariableService.update({ key, value, applicationId });
return true;
}
}
@@ -0,0 +1,58 @@
import { InjectRepository } from '@nestjs/typeorm';
import { In, Not, Repository } from 'typeorm';
import { ApplicationVariable } from 'src/engine/core-modules/applicationVariable/application-variable.entity';
export class ApplicationVariableService {
constructor(
@InjectRepository(ApplicationVariable)
private readonly applicationVariableRepository: Repository<ApplicationVariable>,
) {}
async update({
key,
value,
applicationId,
}: Pick<ApplicationVariable, 'key' | 'value'> & { applicationId: string }) {
await this.applicationVariableRepository.update(
{ key, applicationId },
{
value,
},
);
}
async upsertManyApplicationVariables({
env,
applicationId,
}: {
env: Record<
string,
{
value?: string;
description?: string;
isSecret: boolean;
}
>;
applicationId: string;
}) {
for (const [key, { value, description, isSecret }] of Object.entries(env)) {
await this.applicationVariableRepository.upsert(
{
key,
value,
description,
isSecret,
applicationId,
},
{ conflictPaths: ['key', 'applicationId'] },
);
}
await this.applicationVariableRepository.delete({
applicationId,
key: Not(In(Object.keys(env))),
});
}
}
@@ -0,0 +1,28 @@
import { Field, ObjectType } from '@nestjs/graphql';
import { IsBoolean, IsString } from 'class-validator';
import { IDField } from '@ptc-org/nestjs-query-graphql';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ObjectType('ApplicationVariable')
export class ApplicationVariableDTO {
@IDField(() => UUIDScalarType)
id: string;
@IsString()
@Field()
key: string;
@IsString()
@Field()
value: string;
@IsString()
@Field()
description: string;
@IsBoolean()
@Field()
isSecret: boolean;
}
@@ -0,0 +1,15 @@
import { ArgsType, Field } from '@nestjs/graphql';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ArgsType()
export class UpdateApplicationVariableInput {
@Field(() => String, { nullable: false })
key: string;
@Field(() => String, { nullable: false })
value: string;
@Field(() => UUIDScalarType, { nullable: false })
applicationId: string;
}
@@ -6,17 +6,20 @@ export const handler = async (event) => {
const mainPath = `/tmp/${randomId}.mjs`;
const oldProcessEnv = { ...process.env };
try {
const { code, params } = event;
const { code, params, env } = event;
await fs.writeFile(mainPath, code, 'utf8');
process.env = {};
process.env = { ...process.env, ...(env ?? {}) };
const mainFile = await import(mainPath);
return await mainFile.main(params);
} finally {
await fs.rm(mainPath, { force: true });
process.env = oldProcessEnv;
}
};
@@ -46,6 +46,7 @@ import {
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
@@ -338,6 +339,7 @@ export class LambdaDriver implements ServerlessDriver {
const executorPayload = {
params: payload,
code: compiledCode,
env: buildEnvVar(serverlessFunction),
};
const params: InvokeCommandInput = {
@@ -1,5 +1,6 @@
import { promises as fs } from 'fs';
import { join } from 'path';
import { spawn } from 'node:child_process';
import {
type ServerlessDriver,
@@ -16,6 +17,7 @@ import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serve
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
@@ -162,30 +164,53 @@ export class LocalDriver implements ServerlessDriver {
});
try {
const mainFile = await import(builtBundleFilePath);
const result = await this.executeWithTimeout<object | null>(
() => mainFile.main(payload),
serverlessFunction.timeoutSeconds * 1_000,
const runnerPath = await this.writeBootstrapRunner(
sourceTemporaryDir,
builtBundleFilePath,
);
const { ok, result, error, stack, stdout, stderr } =
await this.runChildWithEnv({
runnerPath,
env: buildEnvVar(serverlessFunction),
payload,
timeoutMs: serverlessFunction.timeoutSeconds * 1_000,
});
if (stdout)
logs +=
stdout
.split('\n')
.filter(Boolean)
.map((l) => `${new Date().toISOString()} INFO ${l}`)
.join('\n') + '\n';
if (stderr)
logs +=
stderr
.split('\n')
.filter(Boolean)
.map((l) => `${new Date().toISOString()} ERROR ${l}`)
.join('\n') + '\n';
const duration = Date.now() - startTime;
return {
data: result,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
if (ok) {
return {
data: (result ?? null) as object | null,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
}
return {
data: null,
logs,
duration: Date.now() - startTime,
duration,
error: {
errorType: 'UnhandledError',
errorMessage: error.message || 'Unknown error',
stackTrace: error.stack ? error.stack.split('\n') : [],
errorMessage: error || 'Unknown error',
stackTrace: stack ? String(stack).split('\n') : [],
},
status: ServerlessFunctionExecutionStatus.ERROR,
};
@@ -196,4 +221,143 @@ export class LocalDriver implements ServerlessDriver {
await lambdaBuildDirectoryManager.clean();
}
}
async writeBootstrapRunner(dir: string, builtFileAbsPath: string) {
const runnerPath = join(dir, '__runner.cjs');
const code = `
// Auto-generated. Do not edit.
const { pathToFileURL } = require('node:url');
(async () => {
try {
const builtUrl = pathToFileURL(${JSON.stringify(builtFileAbsPath)});
const mod = await import(builtUrl.href);
if (typeof mod.main !== 'function') {
throw new Error('Export "main" not found in serverless bundle');
}
let payload = undefined;
if (process.send) {
process.on('message', async (msg) => {
if (!msg || msg.type !== 'run') return;
try {
const out = await mod.main(msg.payload);
process.send && process.send({ ok: true, result: out });
process.exit(0);
} catch (err) {
process.send && process.send({ ok: false, error: String(err), stack: err?.stack });
process.exit(1);
}
});
} else {
// Fallback: read payload from argv[2] (JSON) and print to stdout
const json = process.argv[2];
payload = json ? JSON.parse(json) : undefined;
const out = await mod.main(payload);
console.log(JSON.stringify({ ok: true, result: out }));
process.exit(0);
}
} catch (err) {
const msg = String(err);
if (process.send) {
process.send({ ok: false, error: msg, stack: err?.stack });
} else {
console.error(msg);
}
process.exit(1);
}
})();
`;
await fs.writeFile(runnerPath, code, 'utf8');
return runnerPath;
}
runChildWithEnv(options: {
runnerPath: string;
env: Record<string, string>;
payload: unknown;
timeoutMs: number;
}) {
const { runnerPath, env, payload, timeoutMs } = options;
return new Promise<{
ok: boolean;
result?: unknown;
error?: string;
stack?: string;
stdout: string;
stderr: string;
}>((resolve, _) => {
const child = spawn(process.execPath, [runnerPath], {
env: { ...process.env, ...env },
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
});
let stdout = '';
let stderr = '';
let settled = false;
child.stdout?.on('data', (d) => (stdout += String(d)));
child.stderr?.on('data', (d) => (stderr += String(d)));
child.on(
'message',
(
msg:
| {
ok: true;
result?: unknown;
stdout?: string;
stderr?: string;
}
| {
ok: false;
error: string;
stack?: string;
stdout?: string;
stderr?: string;
},
) => {
if (settled) return;
settled = true;
resolve({ ...msg, stdout, stderr });
},
);
child.on('exit', (code) => {
if (settled) return;
settled = true;
if (code === 0) {
// Fallback path if no IPC (shouldnt happen with our stdio)
resolve({ ok: true, stdout, stderr });
} else {
resolve({
ok: false,
error: `Exited with code ${code}`,
stdout,
stderr,
});
}
});
const t = setTimeout(() => {
if (settled) return;
settled = true;
child.kill('SIGKILL');
resolve({
ok: false,
error: `Timed out after ${timeoutMs}ms`,
stdout,
stderr,
});
}, timeoutMs);
// Kick it off
child.send?.({ type: 'run', payload });
child.on('close', () => clearTimeout(t));
});
}
}
@@ -0,0 +1,12 @@
import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
export const buildEnvVar = (serverlessFunction: ServerlessFunctionEntity) => {
return (serverlessFunction.application?.applicationVariables ?? []).reduce(
(acc, v) => {
acc[v.key] = String(v.value ?? '');
return acc;
},
{} as Record<string, string>,
);
};
@@ -96,7 +96,10 @@ export class ServerlessFunctionService {
id,
workspaceId,
},
relations: ['serverlessFunctionLayer'],
relations: [
'serverlessFunctionLayer',
'application.applicationVariables',
],
});
const resultServerlessFunction = await this.serverlessService.execute(
@@ -4,5 +4,4 @@ export type ServerlessFunctionCode = {
src: {
'index.ts': string;
} & Sources;
'.env'?: string;
};