feat(server): add instance-level file storage layer (#22560)

Part of the app settings architecture cleanup
(twentyhq/core-team-issues#2456) — PR 1 of the instance-level documents
plan. Today all file storage is workspace-scoped
(`FileEntity.workspaceId NOT NULL`, `{workspaceId}/{app}/…` storage
keys, workspace-anchored tokens); instance-level data like
application-registration manifests and tarballs for ownerless catalog
registrations has no first-class home, forcing raw-driver bypasses
(`DefaultAiCatalogService`, prototype #22556).

## Changes (core storage layer only — no HTTP serving, no GraphQL
exposure)

**New `instanceFile` table** (`InstanceFileEntity`) — deliberately
separate from the workspace-scoped `file` table so nothing about the
existing system changes:
- `id`, `path` (unique, `{fileFolder}/{relativePath}` mirroring
FileEntity's convention), `size`, `mimeType`, timestamps
- nullable `applicationRegistrationId` FK (`onDelete: CASCADE`) —
registration-owned documents follow their registration
- no `workspaceId`, no `applicationId`; plain repository (added to the
`prefer-workspace-scoped-repository` lint rule's global-table
exemptions, as the rule's own message directs)

**New `InstanceFileStorageService`** (exported from the global
`FileStorageModule`):
- storage keys under a literal `instance/{fileFolder}/…` prefix —
collision-free with workspace prefixes (UUIDs); scope-validation util
mirroring `validateStoragePathIsWithinWorkspaceOrThrow`
- `writeInstanceFile` (upsert row on `path` conflict + driver write;
throws on failure — no swallowing),
`readInstanceFile`/`readInstanceFileById` (missing file surfaces
`FILE_NOT_FOUND` like `FileStorageService.readFile`),
`checkInstanceFileExists`, `deleteInstanceFile`/`deleteByInstanceFileId`
(bytes best-effort, row authoritative),
`deleteByApplicationRegistrationId` (lifecycle hook for
registration-owned files)
- same driver path as `FileStorageService` (`FileStorageDriverFactory` →
`ValidatedStorageDriver`)

**Migration**: fast instance command `add-instance-file-table` (2.19,
generator-produced; post-command `database:migrate:generate` reports no
pending changes).

## Next PRs in the plan

- PR 2: HTTP serving + token type for instance files (new route + guard;
workspace file endpoints untouched)
- PR 3: application-registration manifests stored as versioned instance
files (supersedes draft #22556)
- PR 4 (optional): registration tarballs migrate to instance scope,
removing the cross-workspace `FileEntity` read in
`application-package-fetcher` and the `ownerWorkspaceId` requirement on
`uploadTarball`

## Verification

- New specs: scope-validation util (traversal cases) + service (upsert
conflict, missing-file error, best-effort byte deletion, registration
cascade) — 16/16; `npx jest "application"` still 31 suites / 160 green
- Typecheck, `lint:diff-with-main`, full `oxfmt --check src/` (6421
files) and full type-aware oxlint clean
- Fast command executed against the local DB — table, unique index, and
CASCADE FK verified via psql; generator then reports no schema drift
- Server boots with the new provider; `generate-metadata-client
--skip-nx-cache` zero diff (no GraphQL change)

---
_Generated by [Claude
Code](https://claude.ai/code/session_011sST4rPLU1Koi2oVGi84ei)_

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/22560?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->
This commit is contained in:
martmull
2026-07-06 12:23:42 +02:00
committed by GitHub
parent 11edd56505
commit 0baf213fa4
15 changed files with 874 additions and 6 deletions
@@ -23,6 +23,7 @@ const STRUCTURAL_EXEMPTIONS = new Set<string>([
'ConnectedAccountEntity',
'ConnectionProviderEntity',
'FrontComponentEntity',
'InstanceFileEntity',
'LogicFunctionEntity',
'MessageFolderEntity',
'RolePermissionFlagEntity',
@@ -0,0 +1,19 @@
import { QueryRunner } from 'typeorm';
import { RegisteredInstanceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-instance-command.decorator';
import { FastInstanceCommand } from 'src/engine/core-modules/upgrade/interfaces/fast-instance-command.interface';
@RegisteredInstanceCommand('2.19.0', 1783240670564)
export class AddInstanceFileTableFastInstanceCommand implements FastInstanceCommand {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query('CREATE TABLE "core"."instanceFile" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "path" text NOT NULL, "size" bigint NOT NULL, "mimeType" character varying NOT NULL DEFAULT \'application/octet-stream\', "applicationRegistrationId" uuid, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, CONSTRAINT "IDX_INSTANCE_FILE_PATH_UNIQUE" UNIQUE ("path"), CONSTRAINT "PK_3d753f7415af93f4dfbd92d58ca" PRIMARY KEY ("id"))');
await queryRunner.query('CREATE INDEX "IDX_INSTANCE_FILE_APPLICATION_REGISTRATION_ID" ON "core"."instanceFile" ("applicationRegistrationId") ');
await queryRunner.query('ALTER TABLE "core"."instanceFile" ADD CONSTRAINT "FK_19422e6d5c43d71b516c10fe755" FOREIGN KEY ("applicationRegistrationId") REFERENCES "core"."applicationRegistration"("id") ON DELETE CASCADE ON UPDATE NO ACTION');
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query('ALTER TABLE "core"."instanceFile" DROP CONSTRAINT "FK_19422e6d5c43d71b516c10fe755"');
await queryRunner.query('DROP INDEX "core"."IDX_INSTANCE_FILE_APPLICATION_REGISTRATION_ID"');
await queryRunner.query('DROP TABLE "core"."instanceFile"');
}
}
@@ -0,0 +1,2 @@
export const ADD_INSTANCE_FILE_TABLE_UPGRADE_COMMAND_NAME =
'2.19.0_AddInstanceFileTableFastInstanceCommand_1783240670564';
@@ -98,6 +98,7 @@ import { AddLogoToApplicationRegistrationFastInstanceCommand } from './2-19/2-19
import { BackfillLogoOnApplicationRegistrationSlowInstanceCommand } from './2-19/2-19-instance-command-slow-1783069673191-backfill-logo-on-application-registration';
import { AddDisplayFieldsToApplicationRegistrationFastInstanceCommand } from './2-19/2-19-instance-command-fast-1783073776590-add-display-fields-to-application-registration';
import { BackfillDisplayFieldsOnApplicationRegistrationSlowInstanceCommand } from './2-19/2-19-instance-command-slow-1783073776591-backfill-display-fields-on-application-registration';
import { AddInstanceFileTableFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-19/2-19-instance-command-fast-1783240670564-add-instance-file-table';
export const INSTANCE_COMMANDS = [
AddViewFieldGroupIdIndexOnViewFieldFastInstanceCommand,
@@ -198,4 +199,5 @@ export const INSTANCE_COMMANDS = [
BackfillDisplayFieldsOnApplicationRegistrationSlowInstanceCommand,
AddStatusToFileFastInstanceCommand,
AddPendingMimeCheckToFileFastInstanceCommand,
AddInstanceFileTableFastInstanceCommand,
];
@@ -0,0 +1,422 @@
import { InstanceFileFolder } from 'twenty-shared/types';
import { Test, type TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { Readable } from 'stream';
import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
import { InstanceFileStorageService } from 'src/engine/core-modules/file-storage/instance-file-storage.service';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { InstanceFileEntity } from 'src/engine/core-modules/file/entities/instance-file.entity';
describe('InstanceFileStorageService', () => {
let service: InstanceFileStorageService;
const mockFileStorageDriverFactory = {
getCurrentDriver: jest.fn(),
};
const mockInstanceFileRepository = {
upsert: jest.fn(),
findOneBy: jest.fn(),
findOneByOrFail: jest.fn(),
findBy: jest.fn(),
delete: jest.fn(),
};
const mockDriver = {
writeFile: jest.fn(),
readFile: jest.fn(),
delete: jest.fn(),
checkFileExists: jest.fn(),
};
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
InstanceFileStorageService,
{
provide: FileStorageDriverFactory,
useValue: mockFileStorageDriverFactory,
},
{
provide: getRepositoryToken(InstanceFileEntity),
useValue: mockInstanceFileRepository,
},
],
}).compile();
service = module.get<InstanceFileStorageService>(
InstanceFileStorageService,
);
jest.clearAllMocks();
mockFileStorageDriverFactory.getCurrentDriver.mockReturnValue(mockDriver);
});
describe.each([
[
'readInstanceFile',
(resourcePath: string) =>
service.readInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath,
}),
],
[
'checkInstanceFileExists',
(resourcePath: string) =>
service.checkInstanceFileExists({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath,
}),
],
[
'deleteInstanceFile',
(resourcePath: string) =>
service.deleteInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath,
}),
],
] as const)('%s traversal protection', (_methodName, invoke) => {
it.each(['../workspace-id/stolen.json', 'a/../../escape.json'])(
'should reject traversal resource path %s without touching storage',
async (resourcePath) => {
await expect(
(async () => {
await invoke(resourcePath);
})(),
).rejects.toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.ACCESS_DENIED,
}),
);
expect(mockDriver.readFile).not.toHaveBeenCalled();
expect(mockDriver.checkFileExists).not.toHaveBeenCalled();
expect(mockDriver.delete).not.toHaveBeenCalled();
},
);
});
describe('writeInstanceFile', () => {
it('should write bytes with the instance prefix and upsert the row on path conflict', async () => {
const instanceFile = {
id: 'instance-file-id',
path: 'application-registration/manifests/manifest.json',
} as InstanceFileEntity;
mockInstanceFileRepository.findOneByOrFail.mockResolvedValue(
instanceFile,
);
const result = await service.writeInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifests/manifest.json',
contents: '{"name":"my-app"}',
mimeType: 'application/json',
applicationRegistrationId: 'registration-id',
});
expect(mockDriver.writeFile).toHaveBeenCalledWith({
filePath: 'instance/application-registration/manifests/manifest.json',
mimeType: 'application/json',
sourceFile: '{"name":"my-app"}',
});
expect(mockInstanceFileRepository.upsert).toHaveBeenCalledWith(
{
path: 'application-registration/manifests/manifest.json',
size: Buffer.byteLength('{"name":"my-app"}'),
mimeType: 'application/json',
applicationRegistrationId: 'registration-id',
},
{ conflictPaths: ['path'] },
);
expect(result).toEqual(instanceFile);
});
it('should store a null applicationRegistrationId when none is provided', async () => {
mockInstanceFileRepository.findOneByOrFail.mockResolvedValue(
{} as InstanceFileEntity,
);
await service.writeInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifest.json',
contents: Buffer.from('{}'),
mimeType: 'application/json',
});
expect(mockInstanceFileRepository.upsert).toHaveBeenCalledWith(
expect.objectContaining({ applicationRegistrationId: null, size: 2 }),
{ conflictPaths: ['path'] },
);
});
it('should reject a traversal resource path without touching storage', async () => {
await expect(
service.writeInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: '../workspace-id/stolen.json',
contents: '{}',
mimeType: 'application/json',
}),
).rejects.toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.ACCESS_DENIED,
}),
);
expect(mockDriver.writeFile).not.toHaveBeenCalled();
expect(mockInstanceFileRepository.upsert).not.toHaveBeenCalled();
});
it('should propagate driver write failures without upserting the row', async () => {
mockDriver.writeFile.mockRejectedValueOnce(new Error('Write failed'));
await expect(
service.writeInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifest.json',
contents: '{}',
mimeType: 'application/json',
}),
).rejects.toThrow('Write failed');
expect(mockInstanceFileRepository.upsert).not.toHaveBeenCalled();
});
});
describe('readInstanceFile', () => {
it('should read from the instance-prefixed storage path', async () => {
const stream = Readable.from(['{}']);
mockInstanceFileRepository.findOneBy.mockResolvedValue({
id: 'instance-file-id',
path: 'application-registration/manifests/manifest.json',
} as InstanceFileEntity);
mockDriver.readFile.mockResolvedValue(stream);
const result = await service.readInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifests/manifest.json',
});
expect(mockDriver.readFile).toHaveBeenCalledWith({
filePath: 'instance/application-registration/manifests/manifest.json',
});
expect(result).toBe(stream);
});
it('should propagate the missing-file exception from the driver', async () => {
mockInstanceFileRepository.findOneBy.mockResolvedValue({
id: 'instance-file-id',
path: 'application-registration/missing.json',
} as InstanceFileEntity);
mockDriver.readFile.mockRejectedValueOnce(
new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
),
);
await expect(
service.readInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'missing.json',
}),
).rejects.toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.FILE_NOT_FOUND,
}),
);
});
it('should throw FILE_NOT_FOUND without reading bytes when the row is missing', async () => {
mockInstanceFileRepository.findOneBy.mockResolvedValue(null);
await expect(
service.readInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'deleted.json',
}),
).rejects.toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.FILE_NOT_FOUND,
}),
);
expect(mockDriver.readFile).not.toHaveBeenCalled();
});
});
describe('readInstanceFileById', () => {
it('should read the bytes of the row storage path', async () => {
const stream = Readable.from(['{}']);
mockInstanceFileRepository.findOneBy.mockResolvedValue({
id: 'instance-file-id',
path: 'application-registration/manifest.json',
} as InstanceFileEntity);
mockDriver.readFile.mockResolvedValue(stream);
const result = await service.readInstanceFileById('instance-file-id');
expect(mockInstanceFileRepository.findOneBy).toHaveBeenCalledWith({
id: 'instance-file-id',
});
expect(mockDriver.readFile).toHaveBeenCalledWith({
filePath: 'instance/application-registration/manifest.json',
});
expect(result).toBe(stream);
});
it('should throw a missing-file exception when the row does not exist', async () => {
mockInstanceFileRepository.findOneBy.mockResolvedValue(null);
await expect(service.readInstanceFileById('unknown-id')).rejects.toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.FILE_NOT_FOUND,
}),
);
expect(mockDriver.readFile).not.toHaveBeenCalled();
});
});
describe('checkInstanceFileExists', () => {
it('should check existence on the instance-prefixed storage path', async () => {
mockDriver.checkFileExists.mockResolvedValue(true);
const result = await service.checkInstanceFileExists({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifest.json',
});
expect(mockDriver.checkFileExists).toHaveBeenCalledWith({
filePath: 'instance/application-registration/manifest.json',
});
expect(result).toBe(true);
});
});
describe('deleteInstanceFile', () => {
it('should delete the bytes and the row', async () => {
await service.deleteInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifests/manifest.json',
});
expect(mockDriver.delete).toHaveBeenCalledWith({
folderPath: 'instance/application-registration/manifests',
filename: 'manifest.json',
});
expect(mockInstanceFileRepository.delete).toHaveBeenCalledWith({
path: 'application-registration/manifests/manifest.json',
});
});
it('should still delete the row when the bytes deletion fails', async () => {
mockDriver.delete.mockRejectedValueOnce(new Error('Delete failed'));
await service.deleteInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifest.json',
});
expect(mockInstanceFileRepository.delete).toHaveBeenCalledWith({
path: 'application-registration/manifest.json',
});
});
it('should propagate row deletion failures', async () => {
mockInstanceFileRepository.delete.mockRejectedValueOnce(
new Error('Row deletion failed'),
);
await expect(
service.deleteInstanceFile({
fileFolder: InstanceFileFolder.ApplicationRegistration,
resourcePath: 'manifest.json',
}),
).rejects.toThrow('Row deletion failed');
});
});
describe('deleteByInstanceFileId', () => {
it('should delete the bytes and the row of the given id', async () => {
mockInstanceFileRepository.findOneBy.mockResolvedValue({
id: 'instance-file-id',
path: 'application-registration/manifest.json',
} as InstanceFileEntity);
await service.deleteByInstanceFileId('instance-file-id');
expect(mockDriver.delete).toHaveBeenCalledWith({
folderPath: 'instance/application-registration',
filename: 'manifest.json',
});
expect(mockInstanceFileRepository.delete).toHaveBeenCalledWith({
id: 'instance-file-id',
});
});
it('should throw a missing-file exception when the row does not exist', async () => {
mockInstanceFileRepository.findOneBy.mockResolvedValue(null);
await expect(
service.deleteByInstanceFileId('unknown-id'),
).rejects.toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.FILE_NOT_FOUND,
}),
);
expect(mockInstanceFileRepository.delete).not.toHaveBeenCalled();
});
});
describe('deleteByApplicationRegistrationId', () => {
it('should delete the bytes of every file then the rows', async () => {
mockInstanceFileRepository.findBy.mockResolvedValue([
{ id: 'file-1', path: 'application-registration/manifest.json' },
{ id: 'file-2', path: 'application-registration/nested/settings.json' },
] as InstanceFileEntity[]);
await service.deleteByApplicationRegistrationId('registration-id');
expect(mockInstanceFileRepository.findBy).toHaveBeenCalledWith({
applicationRegistrationId: 'registration-id',
});
expect(mockDriver.delete).toHaveBeenCalledWith({
folderPath: 'instance/application-registration',
filename: 'manifest.json',
});
expect(mockDriver.delete).toHaveBeenCalledWith({
folderPath: 'instance/application-registration/nested',
filename: 'settings.json',
});
expect(mockInstanceFileRepository.delete).toHaveBeenCalledWith({
applicationRegistrationId: 'registration-id',
});
});
it('should still delete the rows when a bytes deletion fails', async () => {
mockInstanceFileRepository.findBy.mockResolvedValue([
{ id: 'file-1', path: 'application-registration/manifest.json' },
] as InstanceFileEntity[]);
mockDriver.delete.mockRejectedValueOnce(new Error('Delete failed'));
await service.deleteByApplicationRegistrationId('registration-id');
expect(mockInstanceFileRepository.delete).toHaveBeenCalledWith({
applicationRegistrationId: 'registration-id',
});
});
});
});
@@ -0,0 +1 @@
export const INSTANCE_FILE_STORAGE_PREFIX = 'instance';
@@ -6,7 +6,9 @@ import { ApplicationEntity } from 'src/engine/core-modules/application/applicati
import { FileStorageExceptionFilter } from 'src/engine/core-modules/file-storage/file-storage-exception-filter';
import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { InstanceFileStorageService } from 'src/engine/core-modules/file-storage/instance-file-storage.service';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { InstanceFileEntity } from 'src/engine/core-modules/file/entities/instance-file.entity';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/provide-workspace-scoped-repository';
@Global()
@@ -16,18 +18,27 @@ export class FileStorageModule {
module: FileStorageModule,
imports: [
TwentyConfigModule,
TypeOrmModule.forFeature([FileEntity, ApplicationEntity]),
TypeOrmModule.forFeature([
FileEntity,
InstanceFileEntity,
ApplicationEntity,
]),
],
providers: [
FileStorageDriverFactory,
FileStorageService,
InstanceFileStorageService,
provideWorkspaceScopedRepository(FileEntity),
{
provide: APP_FILTER,
useClass: FileStorageExceptionFilter,
},
],
exports: [FileStorageDriverFactory, FileStorageService],
exports: [
FileStorageDriverFactory,
FileStorageService,
InstanceFileStorageService,
],
};
}
}
@@ -0,0 +1,233 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { basename, dirname, join } from 'path';
import { type Readable } from 'stream';
import { type InstanceFileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
import { INSTANCE_FILE_STORAGE_PREFIX } from 'src/engine/core-modules/file-storage/constants/instance-file-storage-prefix.constant';
import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { validateFilePath } from 'src/engine/core-modules/file-storage/utils/validate-file-path.util';
import { validateStoragePathIsWithinInstanceScopeOrThrow } from 'src/engine/core-modules/file-storage/utils/validate-storage-path-is-within-instance-scope-or-throw.util';
import { InstanceFileEntity } from 'src/engine/core-modules/file/entities/instance-file.entity';
export type InstanceResourceIdentifier = {
fileFolder: InstanceFileFolder;
resourcePath: string;
};
@Injectable()
export class InstanceFileStorageService {
private readonly logger = new Logger(InstanceFileStorageService.name);
constructor(
private readonly fileStorageDriverFactory: FileStorageDriverFactory,
@InjectRepository(InstanceFileEntity)
private readonly instanceFileRepository: Repository<InstanceFileEntity>,
) {}
private validateAndBuildInstanceFileStoragePathOrThrow({
fileFolder,
resourcePath,
}: InstanceResourceIdentifier): {
onStorageFilePath: string;
filePath: string;
} {
const validationResult = validateFilePath({ resourcePath, fileFolder });
if (!validationResult.isValid) {
throw new FileStorageException(
validationResult.error,
FileStorageExceptionCode.ACCESS_DENIED,
);
}
const filePath = join(fileFolder, resourcePath).replace(/\/+/g, '/');
const onStorageFilePath = join(
INSTANCE_FILE_STORAGE_PREFIX,
filePath,
).replace(/\/+/g, '/');
validateStoragePathIsWithinInstanceScopeOrThrow({
onStoragePath: onStorageFilePath,
fileFolder,
});
return { onStorageFilePath, filePath };
}
async writeInstanceFile({
fileFolder,
resourcePath,
contents,
mimeType,
applicationRegistrationId,
}: InstanceResourceIdentifier & {
contents: Buffer | string;
mimeType: string;
applicationRegistrationId?: string;
}): Promise<InstanceFileEntity> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const { onStorageFilePath, filePath } =
this.validateAndBuildInstanceFileStoragePathOrThrow({
fileFolder,
resourcePath,
});
await driver.writeFile({
filePath: onStorageFilePath,
mimeType,
sourceFile: contents,
});
await this.instanceFileRepository.upsert(
{
path: filePath,
size:
typeof contents === 'string'
? Buffer.byteLength(contents)
: contents.length,
mimeType,
applicationRegistrationId: applicationRegistrationId ?? null,
},
{ conflictPaths: ['path'] },
);
return this.instanceFileRepository.findOneByOrFail({ path: filePath });
}
async readInstanceFile({
fileFolder,
resourcePath,
}: InstanceResourceIdentifier): Promise<Readable> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const { onStorageFilePath, filePath } =
this.validateAndBuildInstanceFileStoragePathOrThrow({
fileFolder,
resourcePath,
});
const instanceFile = await this.instanceFileRepository.findOneBy({
path: filePath,
});
if (!isDefined(instanceFile)) {
throw new FileStorageException(
`Instance file ${filePath} not found`,
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
return driver.readFile({ filePath: onStorageFilePath });
}
async readInstanceFileById(id: string): Promise<Readable> {
const instanceFile = await this.findInstanceFileByIdOrThrow(id);
const driver = this.fileStorageDriverFactory.getCurrentDriver();
return driver.readFile({
filePath: this.buildOnStorageFilePath(instanceFile),
});
}
checkInstanceFileExists({
fileFolder,
resourcePath,
}: InstanceResourceIdentifier): Promise<boolean> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
const { onStorageFilePath } =
this.validateAndBuildInstanceFileStoragePathOrThrow({
fileFolder,
resourcePath,
});
return driver.checkFileExists({ filePath: onStorageFilePath });
}
async deleteInstanceFile({
fileFolder,
resourcePath,
}: InstanceResourceIdentifier): Promise<void> {
const { onStorageFilePath, filePath } =
this.validateAndBuildInstanceFileStoragePathOrThrow({
fileFolder,
resourcePath,
});
await this.deleteBytesBestEffort(onStorageFilePath);
await this.instanceFileRepository.delete({ path: filePath });
}
async deleteByInstanceFileId(id: string): Promise<void> {
const instanceFile = await this.findInstanceFileByIdOrThrow(id);
await this.deleteBytesBestEffort(this.buildOnStorageFilePath(instanceFile));
await this.instanceFileRepository.delete({ id });
}
async deleteByApplicationRegistrationId(
applicationRegistrationId: string,
): Promise<void> {
const instanceFiles = await this.instanceFileRepository.findBy({
applicationRegistrationId,
});
for (const instanceFile of instanceFiles) {
await this.deleteBytesBestEffort(
this.buildOnStorageFilePath(instanceFile),
);
}
await this.instanceFileRepository.delete({ applicationRegistrationId });
}
private async findInstanceFileByIdOrThrow(
id: string,
): Promise<InstanceFileEntity> {
const instanceFile = await this.instanceFileRepository.findOneBy({ id });
if (!isDefined(instanceFile)) {
throw new FileStorageException(
`Instance file ${id} not found`,
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
return instanceFile;
}
private buildOnStorageFilePath(instanceFile: InstanceFileEntity): string {
return join(INSTANCE_FILE_STORAGE_PREFIX, instanceFile.path);
}
private async deleteBytesBestEffort(
onStorageFilePath: string,
): Promise<void> {
const driver = this.fileStorageDriverFactory.getCurrentDriver();
try {
await driver.delete({
folderPath: dirname(onStorageFilePath),
filename: basename(onStorageFilePath),
});
} catch (error) {
this.logger.warn(
`Failed to delete instance file bytes at ${onStorageFilePath}: ${error}`,
);
}
}
}
@@ -0,0 +1,79 @@
import { InstanceFileFolder } from 'twenty-shared/types';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { validateStoragePathIsWithinInstanceScopeOrThrow } from 'src/engine/core-modules/file-storage/utils/validate-storage-path-is-within-instance-scope-or-throw.util';
const primitives = {
fileFolder: InstanceFileFolder.ApplicationRegistration,
} as const;
describe('validateStoragePathIsWithinInstanceScopeOrThrow', () => {
it.each([
{
title: 'nested path within prefix',
onStoragePath:
'instance/application-registration/manifests/manifest.json',
},
{
title: 'file directly under prefix',
onStoragePath: 'instance/application-registration/manifest.json',
},
])('should accept valid path: $title', ({ onStoragePath }) => {
expect(() =>
validateStoragePathIsWithinInstanceScopeOrThrow({
onStoragePath,
...primitives,
}),
).not.toThrow();
});
it.each([
{
title: 'workspace-like prefix instead of instance prefix',
onStoragePath: 'workspace-id/app-uid/source/file.json',
},
{
title: 'different file folder',
onStoragePath: 'instance/other-folder/file.json',
},
{
title: 'prefix without trailing file',
onStoragePath: 'instance/application-registration',
},
{
title: 'partial prefix match (malicious suffix)',
onStoragePath: 'instance/application-registrationMalicious/file.json',
},
{
title: 'traversal out of the instance prefix',
onStoragePath:
'instance/application-registration/../../workspace-id/file.json',
},
{
title: 'traversal segments kept after normalization',
onStoragePath: 'instance/application-registration/../../../etc/passwd',
},
{
title: 'absolute path',
onStoragePath: '/instance/application-registration/file.json',
},
{
title: 'null byte in path',
onStoragePath: 'instance/application-registration/file\0.json',
},
])(
'should reject path that escapes instance scope: $title',
({ onStoragePath }) => {
expect(() =>
validateStoragePathIsWithinInstanceScopeOrThrow({
onStoragePath,
...primitives,
}),
).toThrow(
expect.objectContaining({
code: FileStorageExceptionCode.ACCESS_DENIED,
}),
);
},
);
});
@@ -1,5 +1,5 @@
import { t } from '@lingui/core/macro';
import { type FileFolder } from 'twenty-shared/types';
import { type FileFolder, type InstanceFileFolder } from 'twenty-shared/types';
import { ALLOWED_EXTENSIONS_BY_APPLICATION_FILE_FOLDER } from 'src/engine/core-modules/file-storage/constants/allowed-extensions-by-application-file-folder.constant';
import { type ResourcePathValidationResult } from 'src/engine/core-modules/file-storage/types/resource-path-validation-result.type';
@@ -10,7 +10,7 @@ export const validateFileExtension = ({
fileFolder,
}: {
resourcePath: string;
fileFolder: FileFolder;
fileFolder: FileFolder | InstanceFileFolder;
}): ResourcePathValidationResult => {
const allowedExtensions =
ALLOWED_EXTENSIONS_BY_APPLICATION_FILE_FOLDER[
@@ -1,5 +1,5 @@
import { t } from '@lingui/core/macro';
import { type FileFolder } from 'twenty-shared/types';
import { type FileFolder, type InstanceFileFolder } from 'twenty-shared/types';
import { type ResourcePathValidationResult } from 'src/engine/core-modules/file-storage/types/resource-path-validation-result.type';
import { validateFileExtension } from 'src/engine/core-modules/file-storage/utils/validate-file-extension.util';
@@ -11,7 +11,7 @@ export const validateFilePath = ({
fileFolder,
}: {
resourcePath: string;
fileFolder: FileFolder;
fileFolder: FileFolder | InstanceFileFolder;
}): ResourcePathValidationResult => {
const safePathResult = validateSafeRelativePath({ resourcePath });
@@ -0,0 +1,32 @@
import { join, normalize } from 'path';
import { type InstanceFileFolder } from 'twenty-shared/types';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { assertStoragePathIsSafe } from 'src/engine/core-modules/file-storage/utils/assert-storage-path-is-safe.util';
import { INSTANCE_FILE_STORAGE_PREFIX } from 'src/engine/core-modules/file-storage/constants/instance-file-storage-prefix.constant';
export const validateStoragePathIsWithinInstanceScopeOrThrow = ({
onStoragePath,
fileFolder,
}: {
onStoragePath: string;
fileFolder: InstanceFileFolder;
}): void => {
assertStoragePathIsSafe(onStoragePath);
const expectedPrefix = join(INSTANCE_FILE_STORAGE_PREFIX, fileFolder);
const normalizedPath = normalize(onStoragePath);
const normalizedPrefix = normalize(expectedPrefix + '/');
if (!normalizedPath.startsWith(normalizedPrefix)) {
throw new FileStorageException(
'Invalid storage path: resolved path escapes the instance scope',
FileStorageExceptionCode.ACCESS_DENIED,
);
}
};
@@ -0,0 +1,62 @@
import {
Column,
CreateDateColumn,
DeleteDateColumn,
Entity,
Index,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
type Relation,
Unique,
UpdateDateColumn,
} from 'typeorm';
import { ADD_INSTANCE_FILE_TABLE_UPGRADE_COMMAND_NAME } from 'src/database/commands/upgrade-version-command/2-19/add-instance-file-table-upgrade-command-name.constant';
import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
import { WasIntroducedInUpgrade } from 'src/engine/core-modules/upgrade/decorators/was-introduced-in-upgrade.decorator';
@WasIntroducedInUpgrade({
upgradeCommandName: ADD_INSTANCE_FILE_TABLE_UPGRADE_COMMAND_NAME,
})
@Entity('instanceFile')
@Index('IDX_INSTANCE_FILE_APPLICATION_REGISTRATION_ID', [
'applicationRegistrationId',
])
@Unique('IDX_INSTANCE_FILE_PATH_UNIQUE', ['path'])
export class InstanceFileEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false, type: 'text' })
path: string;
@Column({ nullable: false, type: 'bigint' })
size: number;
@Column({
nullable: false,
type: 'varchar',
default: 'application/octet-stream',
})
mimeType: string;
@Column({ nullable: true, type: 'uuid' })
applicationRegistrationId: string | null;
@ManyToOne(() => ApplicationRegistrationEntity, {
onDelete: 'CASCADE',
nullable: true,
})
@JoinColumn({ name: 'applicationRegistrationId' })
applicationRegistration: Relation<ApplicationRegistrationEntity> | null;
@CreateDateColumn({ type: 'timestamptz' })
createdAt: Date;
@UpdateDateColumn({ type: 'timestamptz' })
updatedAt: Date;
@DeleteDateColumn({ type: 'timestamptz' })
deletedAt: Date | null;
}
@@ -0,0 +1,3 @@
export enum InstanceFileFolder {
ApplicationRegistration = 'application-registration',
}
@@ -132,6 +132,7 @@ export type { FormatRecordSerializedRelationProperties } from './FormatRecordSer
export type { FromTo } from './FromToType';
export { HTTPMethod } from './HttpMethod';
export type { IndexOf } from './IndexOf.type';
export { InstanceFileFolder } from './InstanceFileFolder';
export type { IsEmptyObject } from './IsEmptyObject.type';
export type { IsEmptyRecord } from './IsEmptyRecord.type';
export type { IsExactly } from './IsExactly';