Backfill package json for custom and standard app (#17681)

# Backfill application package files for custom and standard apps

- Backfill `package.json` / `yarn.lock` (and related fields) for
existing workspaces; new standard/custom apps get default dependency
files.
- Default package files under
`application/constants/default-package-files/`; util with hardcoded
checksums (comment on how to regenerate).
- New **FileFolder.Dependencies** for app dependency files;
**writeFile_v2** accepts optional `queryRunner` for transactional
writes.

Usages:
- **Upgrade command** `upgrade:1-17:backfill-application-package-files`:
standard/custom apps → default files; other apps → from logic function
layer.
- **Workspace creation**: create workspace with
`workspaceCustomApplicationId` first, then create application (enables
same-transaction insert). Migration makes workspace/application/file FKs
deferrable.
-  dev seeder

packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command.ts

@@ -0,0 +1,212 @@
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+
+import { isNonEmptyString } from '@sniptt/guards';
+import { Command } from 'nest-commander';
+import { FileFolder } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
+import { DataSource, Repository } from 'typeorm';
+
+import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
+import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
+import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
+import { parseAvailablePackagesFromPackageJsonAndYarnLock } from 'src/engine/core-modules/application/utils/parse-available-packages-from-package-json-and-yarn-lock.util';
+import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
+import { FileSettings } from 'src/engine/core-modules/file/types/file-settings.types';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { LogicFunctionLayerEntity } from 'src/engine/metadata-modules/logic-function-layer/logic-function-layer.entity';
+import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils';
+import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
+
+@Command({
+  name: 'upgrade:1-17:backfill-application-package-files',
+  description:
+    'Backfill application package files: standard/custom apps get default files, other apps get files from logic function layer',
+})
+export class BackfillApplicationPackageFilesCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(WorkspaceEntity)
+    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
+    protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
+    protected readonly dataSourceService: DataSourceService,
+    protected readonly applicationService: ApplicationService,
+    protected readonly fileStorageService: FileStorageService,
+    protected readonly workspaceCacheService: WorkspaceCacheService,
+    @InjectDataSource()
+    private readonly coreDataSource: DataSource,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Running backfill application package files for workspace ${workspaceId}`,
+    );
+
+    const dryRun = options.dryRun ?? false;
+    const workspace = await this.workspaceRepository.findOne({
+      where: { id: workspaceId },
+      select: ['id', 'workspaceCustomApplicationId'],
+    });
+
+    if (!isDefined(workspace)) {
+      this.logger.warn(`Workspace ${workspaceId} not found, skipping`);
+
+      return;
+    }
+
+    const { twentyStandardFlatApplication, workspaceCustomFlatApplication } =
+      await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
+        { workspaceId },
+      );
+
+    const { flatApplicationMaps } =
+      await this.workspaceCacheService.getOrRecompute(workspaceId, [
+        'flatApplicationMaps',
+      ]);
+
+    const flatApplications = Object.values(
+      flatApplicationMaps.byId,
+    ) as FlatApplication[];
+
+    for (const application of flatApplications) {
+      const isStandardOrCustomApplication =
+        application.id === twentyStandardFlatApplication.id ||
+        application.id === workspaceCustomFlatApplication.id;
+
+      if (isStandardOrCustomApplication) {
+        const needsBackfill =
+          !isNonEmptyString(application.packageJsonFileId) ||
+          !isNonEmptyString(application.yarnLockFileId) ||
+          !isNonEmptyString(application.packageJsonChecksum) ||
+          !isNonEmptyString(application.yarnLockChecksum) ||
+          !isNonEmptyString(application.availablePackages);
+
+        if (needsBackfill) {
+          this.logger.log(
+            `Backfilling standard/custom application ${application.id} with default package files`,
+          );
+
+          if (!dryRun) {
+            await this.applicationService.uploadDefaultPackageFilesAndSetFileIds(
+              application,
+            );
+          }
+        } else {
+          this.logger.log(
+            `Skipping standard/custom application ${application.id} - already has package files`,
+          );
+        }
+
+        continue;
+      }
+
+      const hasLogicFunctionLayer = isDefined(application.logicFunctionLayerId);
+      const needsBackfill =
+        !isNonEmptyString(application.packageJsonFileId) ||
+        !isNonEmptyString(application.yarnLockFileId) ||
+        !isNonEmptyString(application.packageJsonChecksum) ||
+        !isNonEmptyString(application.yarnLockChecksum) ||
+        !isNonEmptyString(application.availablePackages);
+
+      if (!hasLogicFunctionLayer) {
+        this.logger.log(
+          `Skipping application ${application.id} - no logic function layer`,
+        );
+
+        continue;
+      }
+
+      if (!needsBackfill) {
+        this.logger.log(
+          `Skipping application ${application.id} - already has package files`,
+        );
+
+        continue;
+      }
+
+      this.logger.log(
+        `Backfilling application ${application.id} from logic function layer ${application.logicFunctionLayerId}`,
+      );
+
+      if (!dryRun) {
+        await this.backfillApplicationFromLogicFunctionLayer(application);
+      }
+    }
+  }
+
+  private async backfillApplicationFromLogicFunctionLayer(
+    application: Pick<
+      FlatApplication,
+      'id' | 'universalIdentifier' | 'workspaceId' | 'logicFunctionLayerId'
+    >,
+  ): Promise<void> {
+    const layerRepository = this.coreDataSource.getRepository(
+      LogicFunctionLayerEntity,
+    );
+    const layer = await layerRepository.findOne({
+      where: {
+        id: application.logicFunctionLayerId as string,
+        workspaceId: application.workspaceId,
+      },
+      select: ['id', 'packageJson', 'yarnLock'],
+    });
+
+    if (!isDefined(layer)) {
+      this.logger.warn(
+        `Logic function layer ${application.logicFunctionLayerId} not found for application ${application.id}, skipping`,
+      );
+
+      return;
+    }
+
+    const packageJsonContent = JSON.stringify(layer.packageJson, null, 2);
+    const packageJsonChecksum = logicFunctionCreateHash(
+      JSON.stringify(layer.packageJson),
+    );
+    const yarnLockChecksum = logicFunctionCreateHash(layer.yarnLock);
+    const availablePackages = parseAvailablePackagesFromPackageJsonAndYarnLock(
+      packageJsonContent,
+      layer.yarnLock,
+    );
+
+    const dependencyFileSettings: FileSettings = {
+      isTemporaryFile: false,
+      toDelete: false,
+    };
+
+    const packageJsonFile = await this.fileStorageService.writeFile_v2({
+      sourceFile: packageJsonContent,
+      mimeType: undefined,
+      fileFolder: FileFolder.Dependencies,
+      applicationUniversalIdentifier: application.universalIdentifier,
+      workspaceId: application.workspaceId,
+      resourcePath: 'package.json',
+      settings: dependencyFileSettings,
+    });
+
+    const yarnLockFile = await this.fileStorageService.writeFile_v2({
+      sourceFile: layer.yarnLock,
+      mimeType: undefined,
+      fileFolder: FileFolder.Dependencies,
+      applicationUniversalIdentifier: application.universalIdentifier,
+      workspaceId: application.workspaceId,
+      resourcePath: 'yarn.lock',
+      settings: dependencyFileSettings,
+    });
+
+    await this.applicationService.update(application.id, {
+      packageJsonFileId: packageJsonFile.id,
+      yarnLockFileId: yarnLockFile.id,
+      packageJsonChecksum,
+      yarnLockChecksum,
+      availablePackages,
+      workspaceId: application.workspaceId,
+    });
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-upgrade-version-command.module.ts

@@ -1,6 +1,7 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 
+import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command';
 import { DeleteFileRecordsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command';
 import { IdentifyWebhookMetadataCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-identify-webhook-metadata.command';
 import { MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-make-webhook-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -64,6 +65,7 @@ import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objec
     MigrateSendEmailRecipientsCommand,
     MigrateWorkflowCodeStepsCommand,
     SeedWorkflowV1_16Command,
+    BackfillApplicationPackageFilesCommand,
   ],
   exports: [
     MigrateAttachmentToMorphRelationsCommand,
@@ -73,6 +75,7 @@ import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objec
     MigrateSendEmailRecipientsCommand,
     MigrateWorkflowCodeStepsCommand,
     SeedWorkflowV1_16Command,
+    BackfillApplicationPackageFilesCommand,
   ],
 })
 export class V1_17_UpgradeVersionCommandModule {}

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts

@@ -9,6 +9,7 @@ import {
   UpgradeCommandRunner,
   type VersionCommands,
 } from 'src/database/commands/command-runners/upgrade.command-runner';
+import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command';
 import { DeleteFileRecordsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command';
 import { IdentifyWebhookMetadataCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-identify-webhook-metadata.command';
 import { MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-make-webhook-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -34,6 +35,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
     protected readonly dataSourceService: DataSourceService,
 
     // 1.17 Commands
+    protected readonly backfillApplicationPackageFilesCommand: BackfillApplicationPackageFilesCommand,
     protected readonly deleteFileRecordsCommand: DeleteFileRecordsCommand,
     protected readonly migrateAttachmentToMorphRelationsCommand: MigrateAttachmentToMorphRelationsCommand,
     protected readonly identifyWebhookMetadataCommand: IdentifyWebhookMetadataCommand,
@@ -57,6 +59,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
         .makeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand,
       this.migrateWorkflowCodeStepsCommand,
       this.deleteFileRecordsCommand,
+      this.backfillApplicationPackageFilesCommand,
     ];
 
     this.allCommands = {