65155fe50c
Closes twentyhq/core-team-issues#2742 A logic function run is capped by its own `timeoutSeconds` (900s max), so anything that can't finish in one run — a full re-sync, a per-record fan-out, a rate-limited third-party API — had no way to continue. This adds a way to hand that work to the workers. ## What it looks like for an app author ```ts import { enqueueJob } from 'twenty-sdk/logic-function'; await enqueueJob({ logicFunctionUniversalIdentifier: '9f1c3d7e-51b8-4a29-8f0d-7c4e2a6b1d33', payload: { cursor: nextCursor }, retryLimit: 3, priority: 2, delayMs: 60_000, }); ``` The target runs in its own process with its own timeout budget. The classic shape is a function that enqueues *itself* with the next cursor until there is nothing left. ## Changes **twenty-shared** — `EnqueueJobInput` / `EnqueueJobOptions` / `EnqueueJobResult` in `application`. **twenty-server** — new `application-job` module under `core-modules/application`, following the `application-key-value` pattern: - `enqueueJob` mutation on the metadata API, `@AuthApplication`-scoped - the lookup is scoped to `applicationId` + `workspaceId` — that's the authorization boundary, an app can only enqueue its own logic functions, anything else is `LOGIC_FUNCTION_NOT_FOUND` - pushes a `LogicFunctionTriggerJob` onto the existing `logicFunctionQueue`, so the enqueued run goes through the same executor (and the same execution throttling) as every other trigger - the queued run inherits the caller's `userId`/`userWorkspaceId`, so its app access token carries the same permissions as the function that queued it **Job options** are range-checked via `ResolverValidationPipe`, since the values come from application code and an unbounded delay or retry count would let an app pin work in the shared queue: | Option | Default | Range | |--------|---------|-------| | `retryLimit` | `0` | `0`–`10` | | `priority` | queue default | `1`–`10` (lower first) | | `delayMs` | `0` | `0`–7 days | `retryLimit` defaults to `0` rather than inheriting the server-route path's `3`: retries re-run the whole handler, so opting in should be the author's explicit choice. **twenty-sdk** — `enqueueJob` in `twenty-sdk/logic-function`, same shape as `runAgent`/`kv`. **Docs** — new "Background Jobs" page under Extend → Apps → Logic, plus nav and overview entries. **Generated** — regenerated `twenty-front/src/generated-metadata` and `twenty-client-sdk/src/metadata/generated` for the new mutation. ## Tests - `application-job.service.spec.ts` — 5 unit tests: job options mapping, defaults, acting-user propagation, application-scoped lookup, not-found - `enqueue-job.integration-spec.ts` — 5 integration tests: rejects a non-`APPLICATION_ACCESS` token, enqueues a function the app owns, rejects a function owned by another application, rejects an unknown identifier, rejects out-of-range options All green locally, along with `typecheck` for `twenty-server`/`twenty-sdk` and oxlint/oxfmt on the touched files. ## Notes for review - The target is addressed by `universalIdentifier`, matching `runAgent({ agentUniversalIdentifier })` and `ServerRouteDispatchResult.targetLogicFunctionUniversalIdentifier`. Addressing by `name` would be friendlier, but logic function names aren't validated for uniqueness within an app — happy to add it as a convenience if you'd rather. - `enqueueJob` returns as soon as the job is accepted; it can't return the target's result, since the queue driver's `add` returns void. Documented, with a pointer to the KV store for handing results back. --- _Generated by [Claude Code](https://claude.ai/code/session_01QrYvGonS3HMdeuMAVjs5hR)_ <!-- This is an auto-generated description by cubic. --> <a href="https://cubic.dev/pr/twentyhq/twenty/pull/23527?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. --> --------- Co-authored-by: github-actions <github-actions@twenty.com>
200 lines
6.7 KiB
TypeScript
200 lines
6.7 KiB
TypeScript
import { existsSync, mkdirSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
|
|
import gql from 'graphql-tag';
|
|
import { findManyApplications } from 'test/integration/graphql/utils/find-many-applications.util';
|
|
import { generateApplicationToken } from 'test/integration/metadata/suites/application/utils/generate-application-token.util';
|
|
import { createOneLogicFunction } from 'test/integration/metadata/suites/logic-function/utils/create-logic-function.util';
|
|
import { deleteLogicFunction } from 'test/integration/metadata/suites/logic-function/utils/delete-logic-function.util';
|
|
import { executeLogicFunction } from 'test/integration/metadata/suites/logic-function/utils/execute-logic-function.util';
|
|
import { updateLogicFunctionSource } from 'test/integration/metadata/suites/logic-function/utils/update-logic-function-source.util';
|
|
import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';
|
|
import { expectEventually } from 'test/integration/utils/expect-eventually.util';
|
|
import { waitForAllJobsToFinish } from 'test/integration/utils/wait-for-all-jobs-to-finish.util';
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
|
|
import { WORKSPACE_CUSTOM_APPLICATION_NAME } from 'src/engine/core-modules/application/constants/workspace-custom-application.constant';
|
|
import { TWENTY_STANDARD_APPLICATION } from 'src/engine/workspace-manager/twenty-standard-application/constants/twenty-standard-applications';
|
|
|
|
const MARKER_DIRECTORY = join(tmpdir(), `enqueue-job-${uuidv4()}`);
|
|
|
|
const TARGET_SOURCE_CODE = `import { writeFileSync } from 'node:fs';
|
|
|
|
export const main = async (params: { markerPath: string }): Promise<object> => {
|
|
writeFileSync(params.markerPath, 'ran', 'utf-8');
|
|
|
|
return { ok: true };
|
|
};`;
|
|
|
|
const ENQUEUE_JOB = gql`
|
|
mutation EnqueueJob($input: EnqueueJobInput!) {
|
|
enqueueJob(input: $input) {
|
|
enqueued
|
|
logicFunctionUniversalIdentifier
|
|
}
|
|
}
|
|
`;
|
|
|
|
describe('enqueueJob (e2e)', () => {
|
|
let customApplicationToken: string;
|
|
let standardApplicationToken: string;
|
|
let logicFunctionId: string;
|
|
let logicFunctionUniversalIdentifier: string;
|
|
|
|
beforeAll(async () => {
|
|
mkdirSync(MARKER_DIRECTORY, { recursive: true });
|
|
|
|
const { data } = await findManyApplications({ expectToFail: false });
|
|
|
|
const customApplication = data.findManyApplications.find(
|
|
(application) => application.name === WORKSPACE_CUSTOM_APPLICATION_NAME,
|
|
);
|
|
const standardApplication = data.findManyApplications.find(
|
|
(application) =>
|
|
application.universalIdentifier ===
|
|
TWENTY_STANDARD_APPLICATION.universalIdentifier,
|
|
);
|
|
|
|
expect(customApplication).toBeDefined();
|
|
expect(standardApplication).toBeDefined();
|
|
|
|
const [{ data: customTokenData }, { data: standardTokenData }] =
|
|
await Promise.all([
|
|
generateApplicationToken({
|
|
applicationId: customApplication!.id,
|
|
expectToFail: false,
|
|
}),
|
|
generateApplicationToken({
|
|
applicationId: standardApplication!.id,
|
|
expectToFail: false,
|
|
}),
|
|
]);
|
|
|
|
customApplicationToken =
|
|
customTokenData.generateApplicationToken.applicationAccessToken.token;
|
|
standardApplicationToken =
|
|
standardTokenData.generateApplicationToken.applicationAccessToken.token;
|
|
|
|
const { data: createData } = await createOneLogicFunction({
|
|
input: { name: `enqueue-job-target-${uuidv4()}` },
|
|
gqlFields: 'id universalIdentifier',
|
|
expectToFail: false,
|
|
});
|
|
|
|
expect(createData.createOneLogicFunction.universalIdentifier).toBeDefined();
|
|
|
|
logicFunctionId = createData.createOneLogicFunction.id;
|
|
logicFunctionUniversalIdentifier =
|
|
createData.createOneLogicFunction.universalIdentifier!;
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await deleteLogicFunction({
|
|
input: { id: logicFunctionId },
|
|
expectToFail: false,
|
|
});
|
|
|
|
rmSync(MARKER_DIRECTORY, { recursive: true, force: true });
|
|
});
|
|
|
|
it('rejects requests that do not carry an APPLICATION_ACCESS token', async () => {
|
|
const response = await makeMetadataAPIRequest({
|
|
query: ENQUEUE_JOB,
|
|
variables: { input: { logicFunctionUniversalIdentifier } },
|
|
});
|
|
|
|
expect(response.body.errors).toBeDefined();
|
|
expect(response.body.errors[0].message).toContain('APPLICATION_ACCESS');
|
|
});
|
|
|
|
it('enqueues a logic function owned by the calling application and the worker runs it', async () => {
|
|
const markerPath = join(MARKER_DIRECTORY, 'enqueued.txt');
|
|
|
|
await updateLogicFunctionSource({
|
|
input: {
|
|
id: logicFunctionId,
|
|
update: { sourceHandlerCode: TARGET_SOURCE_CODE },
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const { data: buildData } = await executeLogicFunction({
|
|
input: {
|
|
id: logicFunctionId,
|
|
payload: { markerPath: join(MARKER_DIRECTORY, 'build.txt') },
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
expect(buildData.executeOneLogicFunction.error).toBeNull();
|
|
|
|
const response = await makeMetadataAPIRequest(
|
|
{
|
|
query: ENQUEUE_JOB,
|
|
variables: {
|
|
input: { logicFunctionUniversalIdentifier, payload: { markerPath } },
|
|
},
|
|
},
|
|
customApplicationToken,
|
|
);
|
|
|
|
expect(response.body.errors).toBeUndefined();
|
|
expect(response.body.data.enqueueJob).toEqual({
|
|
enqueued: true,
|
|
logicFunctionUniversalIdentifier,
|
|
});
|
|
|
|
await waitForAllJobsToFinish();
|
|
|
|
await expectEventually(() => {
|
|
expect(existsSync(markerPath)).toBe(true);
|
|
});
|
|
});
|
|
|
|
it('rejects a logic function that belongs to another application', async () => {
|
|
const response = await makeMetadataAPIRequest(
|
|
{
|
|
query: ENQUEUE_JOB,
|
|
variables: {
|
|
input: { logicFunctionUniversalIdentifier },
|
|
},
|
|
},
|
|
standardApplicationToken,
|
|
);
|
|
|
|
expect(response.body.errors).toBeDefined();
|
|
expect(response.body.errors[0].message).toContain('not found');
|
|
});
|
|
|
|
it('rejects an unknown logic function', async () => {
|
|
const response = await makeMetadataAPIRequest(
|
|
{
|
|
query: ENQUEUE_JOB,
|
|
variables: {
|
|
input: { logicFunctionUniversalIdentifier: uuidv4() },
|
|
},
|
|
},
|
|
customApplicationToken,
|
|
);
|
|
|
|
expect(response.body.errors).toBeDefined();
|
|
expect(response.body.errors[0].message).toContain('not found');
|
|
});
|
|
|
|
it('rejects job options outside of their allowed range', async () => {
|
|
const response = await makeMetadataAPIRequest(
|
|
{
|
|
query: ENQUEUE_JOB,
|
|
variables: {
|
|
input: { logicFunctionUniversalIdentifier, retryLimit: 99 },
|
|
},
|
|
},
|
|
customApplicationToken,
|
|
);
|
|
|
|
expect(response.body.errors).toBeDefined();
|
|
expect(response.body.errors[0].message).toContain('retryLimit');
|
|
});
|
|
});
|