houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backfill package json for custom and standard app (#17681)

Browse Source 
# Backfill application package files for custom and standard apps

- Backfill `package.json` / `yarn.lock` (and related fields) for
existing workspaces; new standard/custom apps get default dependency
files.
- Default package files under
`application/constants/default-package-files/`; util with hardcoded
checksums (comment on how to regenerate).
- New **FileFolder.Dependencies** for app dependency files;
**writeFile_v2** accepts optional `queryRunner` for transactional
writes.

Usages:
- **Upgrade command** `upgrade:1-17:backfill-application-package-files`:
standard/custom apps → default files; other apps → from logic function
layer.
- **Workspace creation**: create workspace with
`workspaceCustomApplicationId` first, then create application (enables
same-transaction insert). Migration makes workspace/application/file FKs
deferrable.
-  dev seeder
This commit is contained in:
Charles Bochet
2026-02-03 21:23:29 +01:00
committed by GitHub
parent 60b48339b9
commit 867b03393b
18 changed files with 3938 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/core-modules/application/constants/default-package-files/package.json
+45
View File
@@ -0,0 +1,45 @@
{
"dependencies": {
"@types/bcrypt": "^5.0.2",
"@types/deep-equal": "^1.0.4",
"@types/lodash.camelcase": "^4.3.9",
"@types/lodash.compact": "^3.0.9",
"@types/lodash.groupby": "^4.6.9",
"@types/lodash.identity": "^3.0.9",
"@types/lodash.isempty": "^4.4.9",
"@types/lodash.isequal": "^4.5.8",
"@types/lodash.isobject": "^3.0.9",
"@types/lodash.kebabcase": "^4.1.9",
"@types/lodash.mapvalues": "^4.6.9",
"@types/lodash.omit": "^4.5.9",
"@types/lodash.pickby": "^4.6.9",
"@types/lodash.snakecase": "^4.1.9",
"@types/lodash.upperfirst": "^4.3.9",
"@types/uuid": "^10.0.0",
"archiver": "^7.0.1",
"axios": "^1.12.0",
"bcrypt": "^5.1.1",
"body-parser": "^1.20.4",
"deep-equal": "^2.2.3",
"jsonwebtoken": "^9.0.2",
"lodash.camelcase": "^4.3.0",
"lodash.chunk": "^4.2.0",
"lodash.compact": "^3.0.1",
"lodash.groupby": "^4.6.0",
"lodash.identity": "^3.0.0",
"lodash.isempty": "^4.4.0",
"lodash.isequal": "^4.5.0",
"lodash.isobject": "^3.0.2",
"lodash.kebabcase": "^4.1.1",
"lodash.mapvalues": "^4.6.0",
"lodash.merge": "^4.6.2",
"lodash.omit": "^4.5.0",
"lodash.pickby": "^4.6.0",
"lodash.snakecase": "^4.1.1",
"lodash.upperfirst": "^4.3.1",
"nodemailer": "^7.0.11",
"sharp": "^0.33.5",
"uuid": "^10.0.0",
"winston": "^3.14.2"
}
}
packages/twenty-server/src/engine/core-modules/application/constants/default-package-files/yarn.lock
+3374
View File
File diff suppressed because it is too large Load Diff
packages/twenty-server/src/engine/core-modules/application/services/application-sync.service.ts
+24 -8
View File
@@ -23,7 +23,9 @@ import {
import { ApplicationInput } from 'src/engine/core-modules/application/dtos/application.input';
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { getDefaultApplicationPackageFields } from 'src/engine/core-modules/application/utils/get-default-application-package-fields.util';
import { ApplicationVariableEntityService } from 'src/engine/core-modules/applicationVariable/application-variable.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LogicFunctionLayerService } from 'src/engine/core-modules/logic-function/logic-function-layer/services/logic-function-layer.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
@@ -43,7 +45,6 @@ import { PermissionFlagService } from 'src/engine/metadata-modules/permission-fl
import { RoleService } from 'src/engine/metadata-modules/role/role.service';
import { computeMetadataNameFromLabelOrThrow } from 'src/engine/metadata-modules/utils/compute-metadata-name-from-label-or-throw.util';
import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
@Injectable()
@@ -147,12 +148,15 @@ export class ApplicationSyncService {
).toString('utf-8'),
) as PackageJson;
const application =
(await this.applicationService.findByUniversalIdentifier({
universalIdentifier: manifest.application.universalIdentifier,
workspaceId,
})) ??
(await this.applicationService.create({
const defaultPackageFields = await getDefaultApplicationPackageFields();
let application = await this.applicationService.findByUniversalIdentifier({
universalIdentifier: manifest.application.universalIdentifier,
workspaceId,
});
if (!application) {
const created = await this.applicationService.create({
universalIdentifier: manifest.application.universalIdentifier,
name,
description: manifest.application.description,
@@ -161,7 +165,19 @@ export class ApplicationSyncService {
logicFunctionLayerId: null,
defaultRoleId: null,
workspaceId,
}));
packageJsonChecksum: defaultPackageFields.packageJsonChecksum,
packageJsonFileId: null,
yarnLockChecksum: defaultPackageFields.yarnLockChecksum,
yarnLockFileId: null,
availablePackages: defaultPackageFields.availablePackages,
});
await this.applicationService.uploadDefaultPackageFilesAndSetFileIds(
created,
);
application = created;
}
let logicFunctionLayerId = application.logicFunctionLayerId;
packages/twenty-server/src/engine/core-modules/application/services/application.service.ts
+99 -3
View File
@@ -1,17 +1,21 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { type QueryRunner, type Repository } from 'typeorm';
import { v4 } from 'uuid';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import {
ApplicationException,
ApplicationExceptionCode,
} from 'src/engine/core-modules/application/application.exception';
import { getDefaultApplicationPackageFields } from 'src/engine/core-modules/application/utils/get-default-application-package-fields.util';
import { parseAvailablePackagesFromPackageJsonAndYarnLock } from 'src/engine/core-modules/application/utils/parse-available-packages-from-package-json-and-yarn-lock.util';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { ALL_FLAT_ENTITY_MAPS_PROPERTIES } from 'src/engine/metadata-modules/flat-entity/constant/all-flat-entity-maps-properties.constant';
import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { TWENTY_STANDARD_APPLICATION } from 'src/engine/workspace-manager/twenty-standard-application/constants/twenty-standard-applications';
@@ -21,6 +25,7 @@ export class ApplicationService {
@InjectRepository(ApplicationEntity)
private readonly applicationRepository: Repository<ApplicationEntity>,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly fileStorageService: FileStorageService,
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
) {}
@@ -261,16 +266,28 @@ export class ApplicationService {
},
queryRunner?: QueryRunner,
) {
const defaultPackageFields = await getDefaultApplicationPackageFields();
const twentyStandardApplication = await this.create(
{
...TWENTY_STANDARD_APPLICATION,
logicFunctionLayerId: null,
workspaceId,
canBeUninstalled: false,
packageJsonChecksum: defaultPackageFields.packageJsonChecksum,
packageJsonFileId: null,
yarnLockChecksum: defaultPackageFields.yarnLockChecksum,
yarnLockFileId: null,
availablePackages: defaultPackageFields.availablePackages,
},
queryRunner,
);
await this.uploadDefaultPackageFilesAndSetFileIds(
twentyStandardApplication,
queryRunner,
);
if (!skipCacheInvalidation) {
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'flatApplicationMaps',
@@ -283,14 +300,17 @@ export class ApplicationService {
async createWorkspaceCustomApplication(
{
workspaceId,
applicationId,
workspaceDisplayName,
}: {
workspaceId: string;
applicationId: string;
workspaceDisplayName?: string;
},
queryRunner?: QueryRunner,
) {
const applicationId = v4();
const defaultPackageFields = await getDefaultApplicationPackageFields();
const workspaceCustomApplication = await this.create(
{
description: 'Workspace custom application',
@@ -298,17 +318,93 @@ export class ApplicationService {
sourcePath: 'workspace-custom',
version: '1.0.0',
universalIdentifier: applicationId,
workspaceId: workspaceId,
workspaceId,
id: applicationId,
logicFunctionLayerId: null,
canBeUninstalled: false,
packageJsonChecksum: defaultPackageFields.packageJsonChecksum,
packageJsonFileId: null,
yarnLockChecksum: defaultPackageFields.yarnLockChecksum,
yarnLockFileId: null,
availablePackages: defaultPackageFields.availablePackages,
},
queryRunner,
);
await this.uploadDefaultPackageFilesAndSetFileIds(
workspaceCustomApplication,
queryRunner,
);
return workspaceCustomApplication;
}
async uploadDefaultPackageFilesAndSetFileIds(
application: Pick<
ApplicationEntity,
'id' | 'universalIdentifier' | 'workspaceId'
>,
queryRunner?: QueryRunner,
): Promise<void> {
const defaultPackageFields = await getDefaultApplicationPackageFields();
const packageJsonChecksum = logicFunctionCreateHash(
JSON.stringify(JSON.parse(defaultPackageFields.packageJsonContent)),
);
const yarnLockChecksum = logicFunctionCreateHash(
defaultPackageFields.yarnLockContent,
);
const availablePackages = parseAvailablePackagesFromPackageJsonAndYarnLock(
defaultPackageFields.packageJsonContent,
defaultPackageFields.yarnLockContent,
);
const packageJsonFile = await this.fileStorageService.writeFile_v2({
sourceFile: defaultPackageFields.packageJsonContent,
mimeType: undefined,
fileFolder: FileFolder.Dependencies,
applicationUniversalIdentifier: application.universalIdentifier,
workspaceId: application.workspaceId,
resourcePath: 'package.json',
settings: { isTemporaryFile: false, toDelete: false },
queryRunner,
});
const yarnLockFile = await this.fileStorageService.writeFile_v2({
sourceFile: defaultPackageFields.yarnLockContent,
mimeType: undefined,
fileFolder: FileFolder.Dependencies,
applicationUniversalIdentifier: application.universalIdentifier,
workspaceId: application.workspaceId,
resourcePath: 'yarn.lock',
settings: { isTemporaryFile: false, toDelete: false },
queryRunner,
});
if (queryRunner) {
await queryRunner.manager.update(
ApplicationEntity,
{ id: application.id },
{
packageJsonFileId: packageJsonFile.id,
yarnLockFileId: yarnLockFile.id,
packageJsonChecksum,
yarnLockChecksum,
availablePackages,
},
);
} else {
await this.update(application.id, {
packageJsonFileId: packageJsonFile.id,
yarnLockFileId: yarnLockFile.id,
packageJsonChecksum,
yarnLockChecksum,
availablePackages,
workspaceId: application.workspaceId,
});
}
}
async create(
data: Partial<ApplicationEntity> & { workspaceId: string },
queryRunner?: QueryRunner,
packages/twenty-server/src/engine/core-modules/application/utils/get-default-application-package-fields.util.ts
+45
View File
@@ -0,0 +1,45 @@
import { readFile } from 'fs/promises';
import path from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
import { parseAvailablePackagesFromPackageJsonAndYarnLock } from 'src/engine/core-modules/application/utils/parse-available-packages-from-package-json-and-yarn-lock.util';
const DEFAULT_PACKAGE_FILES_DIR = path.join(
ASSET_PATH,
'engine/core-modules/application/constants/default-package-files',
);
// To regenerate: use logicFunctionCreateHash from logic-function-create-hash.utils.
// package.json: hash(JSON.stringify(JSON.parse(content))). yarn.lock: hash(content).
// Both use first 32 chars of SHA512 hex digest.
const DEFAULT_PACKAGE_JSON_CHECKSUM = '84772134573e316a4eb1f9dc2e58706a';
const DEFAULT_YARN_LOCK_CHECKSUM = '8d837b7503cf8eff21c6446a126588d4';
export type DefaultApplicationPackageFields = {
packageJsonChecksum: string;
yarnLockChecksum: string;
availablePackages: Record<string, string>;
packageJsonContent: string;
yarnLockContent: string;
};
export const getDefaultApplicationPackageFields =
async (): Promise<DefaultApplicationPackageFields> => {
const [packageJsonContent, yarnLockContent] = await Promise.all([
readFile(path.join(DEFAULT_PACKAGE_FILES_DIR, 'package.json'), 'utf8'),
readFile(path.join(DEFAULT_PACKAGE_FILES_DIR, 'yarn.lock'), 'utf8'),
]);
const availablePackages = parseAvailablePackagesFromPackageJsonAndYarnLock(
packageJsonContent,
yarnLockContent,
);
return {
packageJsonChecksum: DEFAULT_PACKAGE_JSON_CHECKSUM,
yarnLockChecksum: DEFAULT_YARN_LOCK_CHECKSUM,
availablePackages,
packageJsonContent,
yarnLockContent,
};
};
packages/twenty-server/src/engine/core-modules/application/utils/parse-available-packages-from-package-json-and-yarn-lock.util.ts
+31
View File
@@ -0,0 +1,31 @@
import { type PackageJson } from 'type-fest';
const PACKAGE_VERSION_REGEX =
/^"(@?[^@]+(?:\/[^@]+)?)@.*?":\n\s+version:\s*(.+)$/gm;
const MAX_PACKAGE_VERSION_MATCHES = 1_000;
export const parseAvailablePackagesFromPackageJsonAndYarnLock = (
packageJsonContent: string,
yarnLockContent: string,
): Record<string, string> => {
const packageJson = JSON.parse(packageJsonContent) as PackageJson;
const versions: Record<string, string> = {};
let match: RegExpExecArray | null;
let matchCount = 0;
while (
matchCount < MAX_PACKAGE_VERSION_MATCHES &&
(match = PACKAGE_VERSION_REGEX.exec(yarnLockContent)) !== null
) {
matchCount += 1;
const packageName = match[1];
const version = match[2];
if (packageJson.dependencies?.[packageName]) {
versions[packageName] = version;
}
}
return versions;
};
packages/twenty-server/src/engine/core-modules/auth/services/sign-in-up.service.ts
+10 -9
View File
@@ -465,26 +465,19 @@ export class SignInUpService {
isWorkEmailFound && (await isLogoUrlValid()) ? logoUrl : undefined;
const workspaceId = v4();
const workspaceCustomApplicationId = v4();
const queryRunner = this.dataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
try {
const workspaceCustomApplication =
await this.applicationService.createWorkspaceCustomApplication(
{
workspaceId,
},
queryRunner,
);
const workspaceToCreate = this.workspaceRepository.create({
id: workspaceId,
subdomain: await this.subdomainManagerService.generateSubdomain(
isWorkEmailFound ? { userEmail: email } : {},
),
workspaceCustomApplicationId: workspaceCustomApplication.id,
workspaceCustomApplicationId,
displayName: '',
inviteHash: v4(),
activationStatus: WorkspaceActivationStatus.PENDING_CREATION,
@@ -496,6 +489,14 @@ export class SignInUpService {
workspaceToCreate,
);
await this.applicationService.createWorkspaceCustomApplication(
{
workspaceId,
applicationId: workspaceCustomApplicationId,
},
queryRunner,
);
const isExistingUser = userData.type === 'existingUser';
const user = isExistingUser
? userData.existingUser
packages/twenty-server/src/engine/core-modules/file-storage/file-storage.service.ts
+13 -4
View File
@@ -7,7 +7,7 @@ import { type Readable } from 'stream';
import { isObject } from '@sniptt/guards';
import { FileFolder, Sources } from 'twenty-shared/types';
import { Like, Repository } from 'typeorm';
import { Like, Repository, type QueryRunner } from 'typeorm';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
@@ -69,15 +69,24 @@ export class FileStorageService {
resourcePath,
fileId,
settings,
queryRunner,
}: ResourceIdentifier & {
sourceFile: string | Buffer | Uint8Array;
mimeType: string | undefined;
fileId?: string;
settings: FileSettings;
queryRunner?: QueryRunner;
}): Promise<FileEntity> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const application = await this.applicationRepository.findOneOrFail({
const applicationRepository = queryRunner
? queryRunner.manager.getRepository(ApplicationEntity)
: this.applicationRepository;
const fileRepository = queryRunner
? queryRunner.manager.getRepository(FileEntity)
: this.fileRepository;
const application = await applicationRepository.findOneOrFail({
where: {
universalIdentifier: applicationUniversalIdentifier,
workspaceId,
@@ -97,7 +106,7 @@ export class FileStorageService {
sourceFile,
});
await this.fileRepository.upsert(
await fileRepository.upsert(
{
path: `${fileFolder}/${resourcePath}`,
workspaceId,
@@ -112,7 +121,7 @@ export class FileStorageService {
['path', 'workspaceId', 'applicationId'],
);
return await this.fileRepository.findOneOrFail({
return await fileRepository.findOneOrFail({
where: {
path: `${fileFolder}/${resourcePath}`,
applicationId: application.id,
packages/twenty-server/src/engine/core-modules/file/interfaces/file-folder.interface.ts
+3
View File
@@ -45,6 +45,9 @@ export const fileFolderConfigs: Record<FileFolder, FileFolderConfig> = {
[FileFolder.FilesField]: {
ignoreExpirationToken: false,
},
[FileFolder.Dependencies]: {
ignoreExpirationToken: false,
},
};
export type AllowedFolders = KebabCase<keyof typeof FileFolder>;