twenty/.github/workflows/ci-server.yaml

name: CI Server

on:
  pull_request:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

env:
  SERVER_BUILD_CACHE_KEY: server-build

jobs:
  changed-files-check:
    uses: ./.github/workflows/changed-files.yaml
    with:
      files: |
        package.json
        yarn.lock
        packages/twenty-server/**
        packages/twenty-front/src/generated/**
        packages/twenty-front/src/generated-metadata/**
        packages/twenty-front/src/generated-admin/**
        packages/twenty-client-sdk/**
        packages/twenty-emails/**
        packages/twenty-shared/**

  upgrade-changed-files-check:
    uses: ./.github/workflows/changed-files.yaml
    with:
      files: |
        packages/twenty-server/src/database/commands/upgrade-version-command/**
        packages/twenty-server/src/engine/core-modules/upgrade/**

  server-build:
    needs: changed-files-check
    if: needs.changed-files-check.outputs.any_changed == 'true'
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - name: Fetch custom Github Actions and base branch history
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Install dependencies
        uses: ./.github/actions/yarn-install
      - name: Restore server build cache
        id: restore-server-build-cache
        uses: ./.github/actions/restore-cache
        with:
          key: ${{ env.SERVER_BUILD_CACHE_KEY }}
      - name: Build twenty-shared
        run: npx nx build twenty-shared
      - name: Server / Write .env
        run: npx nx reset:env twenty-server
      - name: Server / Build
        run: npx nx build twenty-server
      - name: Save server build cache
        uses: ./.github/actions/save-cache
        with:
          key: ${{ steps.restore-server-build-cache.outputs.cache-primary-key }}

  server-lint-typecheck:
    needs: changed-files-check
    if: needs.changed-files-check.outputs.any_changed == 'true'
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - name: Fetch custom Github Actions and base branch history
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Install dependencies
        uses: ./.github/actions/yarn-install
      - name: Build twenty-shared
        run: npx nx build twenty-shared
      - name: Server / Run lint, typecheck & lingui extract
        uses: ./.github/actions/nx-affected
        with:
          tag: scope:backend
          tasks: lint,typecheck,lingui:extract
      # Separate step so compile never races extract on the .po files.
      - name: Server / Run lingui compile
        uses: ./.github/actions/nx-affected
        with:
          tag: scope:backend
          tasks: lingui:compile

  server-previous-version-upgrade-mutation-guard:
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Fetch custom Github Actions and base branch history
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Get changed upgrade-version-command files
        id: changed-files
        uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v45.0.9
        with:
          files: |
            packages/twenty-server/src/database/commands/upgrade-version-command/**
      - name: Check upgrade version commands are in current version only
        if: >
          steps.changed-files.outputs.any_changed == 'true' &&
          !contains(github.event.pull_request.labels.*.name, 'ci:allow-previous-version-upgrade-mutation')
        run: |
          VERSION_CONSTANT_FILE="packages/twenty-server/src/engine/core-modules/upgrade/constants/twenty-current-version.constant.ts"

          CURRENT_VERSION=$(sed -n "s/.*TWENTY_CURRENT_VERSION = '\([0-9.]*\)'.*/\1/p" "$VERSION_CONSTANT_FILE")

          if [ -z "$CURRENT_VERSION" ]; then
            echo "::error::Could not extract TWENTY_CURRENT_VERSION from $VERSION_CONSTANT_FILE"
            exit 1
          fi

          CURRENT_DIR=$(echo "$CURRENT_VERSION" | sed -E 's/^([0-9]+)\.([0-9]+)\..*/\1-\2/')

          echo "Current version: $CURRENT_VERSION (directory: $CURRENT_DIR)"

          ADDED_OFFENDERS=""
          MODIFIED_OFFENDERS=""

          check_files() {
            local category="$1"
            shift
            for file in "$@"; do
              VERSION_DIR=$(echo "$file" | sed -n 's|.*upgrade-version-command/\([0-9]*-[0-9]*\)/.*|\1|p')

              if [ -n "$VERSION_DIR" ] && [ "$VERSION_DIR" != "$CURRENT_DIR" ]; then
                if [ "$category" = "added" ]; then
                  ADDED_OFFENDERS="$ADDED_OFFENDERS\n  - $file (version directory: $VERSION_DIR)"
                else
                  MODIFIED_OFFENDERS="$MODIFIED_OFFENDERS\n  - $file (version directory: $VERSION_DIR)"
                fi
              fi
            done
          }

          check_files "added" ${{ steps.changed-files.outputs.added_files }}
          check_files "modified" ${{ steps.changed-files.outputs.modified_files }}

          if [ -n "$ADDED_OFFENDERS" ] || [ -n "$MODIFIED_OFFENDERS" ]; then
            echo "This PR touches upgrade command files outside the current version directory ($CURRENT_DIR / $CURRENT_VERSION)."

            if [ -n "$ADDED_OFFENDERS" ]; then
              echo ""
              echo "New files added to non-current version directories:"
              echo -e "$ADDED_OFFENDERS"
            fi

            if [ -n "$MODIFIED_OFFENDERS" ]; then
              echo ""
              echo "Existing files modified in non-current version directories:"
              echo -e "$MODIFIED_OFFENDERS"
            fi

            echo ""
            echo "If this is intentional, add the label 'ci:allow-previous-version-upgrade-mutation' to this PR and re-run CI."
            echo "Otherwise, please move your changes to the current version directory ($CURRENT_DIR)."

            echo "::error::Upgrade commands were added or modified in non-current version directories."
            exit 1
          fi
      - name: Check upgrade command timestamps are real and keep the sequence append-only
        if: >
          steps.changed-files.outputs.any_changed == 'true' &&
          !contains(github.event.pull_request.labels.*.name, 'ci:allow-upgrade-command-timestamp-exception')
        run: |
          BASE_SHA="${{ github.event.pull_request.base.sha }}"

          if [ -z "$BASE_SHA" ]; then
            echo "Not a pull request context, skipping timestamp checks."
            exit 0
          fi

          git fetch --depth=1 origin "$BASE_SHA"

          COMMANDS_ROOT="packages/twenty-server/src/database/commands/upgrade-version-command"

          extract_ts() {
            basename "$1" | sed -n -E 's/.*-(instance-command-(fast|slow)|workspace-command)-([0-9]{13})-.*/\3/p'
          }

          DIFF=$(git diff --name-status --find-renames "$BASE_SHA" HEAD -- "$COMMANDS_ROOT")

          NEW_FILES=$(echo "$DIFF" | awk '$1 == "A" { print $2 } $1 ~ /^[RC]/ { print $3 }')
          GONE_FILES=$(echo "$DIFF" | awk '$1 == "D" { print $2 } $1 ~ /^R/ { print $2 }')

          NOW_MS=$(($(date +%s) * 1000))
          MIN_TS=$((NOW_MS - 60 * 24 * 3600 * 1000))
          MAX_TS=$((NOW_MS + 2 * 24 * 3600 * 1000))

          FAILED=""

          for file in $NEW_FILES; do
            case "$file" in *.spec.ts) continue ;; esac

            TS=$(extract_ts "$file")

            if [ -z "$TS" ]; then
              continue
            fi

            if [ "$TS" -lt "$MIN_TS" ] || [ "$TS" -gt "$MAX_TS" ]; then
              FAILED="$FAILED\n  - $file (timestamp $TS is outside [now - 60 days, now + 2 days])"
              continue
            fi

            VERSION_DIR=$(dirname "$file")

            MAX_BASE_TS=0
            for base_file in $(git ls-tree -r --name-only "$BASE_SHA" -- "$VERSION_DIR"); do
              case "$base_file" in *.spec.ts) continue ;; esac

              if echo "$GONE_FILES" | grep -qxF "$base_file"; then
                continue
              fi

              BASE_TS=$(extract_ts "$base_file")

              if [ -n "$BASE_TS" ] && [ "$BASE_TS" -gt "$MAX_BASE_TS" ]; then
                MAX_BASE_TS=$BASE_TS
              fi
            done

            if [ "$MAX_BASE_TS" -gt 0 ] && [ "$TS" -le "$MAX_BASE_TS" ]; then
              FAILED="$FAILED\n  - $file (timestamp $TS <= existing max $MAX_BASE_TS)"
            fi
          done

          if [ -n "$FAILED" ]; then
            echo "Upgrade command timestamps must be the real epoch millis of when the command"
            echo "is authored (within 60 days before / 2 days after now), and strictly greater"
            echo "than every existing command in the same version directory."
            echo ""
            echo "The upgrade cursor tracks a single last-applied position: a command that sorts"
            echo "before already-applied commands moves the cursor backwards when it runs,"
            echo "re-hiding columns gated by @WasIntroducedInUpgrade on instances that already"
            echo "applied them. Fabricated future timestamps poison the sequence for every"
            echo "command added after them."
            echo ""
            echo "If the blocking existing max is itself a fabricated future timestamp, re-slot"
            echo "that command to its real merge epoch (rename the file and the timestamp in the"
            echo "decorator/constant, and make its up() idempotent since it will re-run under"
            echo "the new name) instead of inflating the new command's timestamp."
            echo ""
            echo "To bypass this check for a deliberate exception, add the label"
            echo "'ci:allow-upgrade-command-timestamp-exception' to this PR and re-run CI."
            echo -e "$FAILED"
            echo "::error::Upgrade command timestamps must be real (close to now) and append-only within their version directory."
            exit 1
          fi

  server-validation:
    needs: server-build
    timeout-minutes: 30
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:18
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - name: Fetch custom Github Actions and base branch history
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Install dependencies
        uses: ./.github/actions/yarn-install
      - name: Restore server build cache
        uses: ./.github/actions/restore-cache
        with:
          key: ${{ env.SERVER_BUILD_CACHE_KEY }}
      - name: Build twenty-shared
        run: npx nx build twenty-shared
      - name: Server / Write .env
        run: npx nx reset:env twenty-server
      - name: Server / Build
        run: npx nx build twenty-server
      - name: Server / Create DB
        run: |
          PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
          PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
          npx nx run twenty-server:database:init:prod
      - name: Worker / Run
        run: |
          timeout 30s npx nx run twenty-server:worker || exit_code=$?
          if [ $exit_code -eq 124 ]; then
            exit 0
          elif [ $exit_code -ne 0 ]; then
            exit $exit_code
          fi
      - name: Server / Start
        run: npx nx start:ci twenty-server &
      - name: Waiting for server starting...
        run: |
          for i in {1..10}; do
            if curl -f http://localhost:3000/healthz; then
              echo "Server ready!"
              exit 0
            fi
            echo "Waiting..."
            sleep 2
          done
          echo "Server did not become healthy in time" >&2
          exit 1
      - name: Server / Check for Pending Migrations
        run: |
          npx nx database:migrate:generate twenty-server -- --name pending-migration-check || true

          if ! git diff --quiet; then
            echo "::error::Unexpected migration files were generated. Please run 'npx nx database:migrate:generate twenty-server -- --name <migration-name>' and commit the result."
            echo ""
            echo "The following migration changes were detected:"
            echo "==================================================="
            git diff
            echo "==================================================="
            echo ""

            git checkout -- .

            exit 1
          fi
      - name: Check for Pending Code Generation
        run: |
          HAS_ERRORS=false

          npx nx run twenty-front:graphql:generate
          npx nx run twenty-front:graphql:generate --configuration=metadata
          npx nx run twenty-front:graphql:generate --configuration=admin

          if ! git diff --quiet -- packages/twenty-front/src/generated packages/twenty-front/src/generated-metadata packages/twenty-front/src/generated-admin; then
            echo "::error::GraphQL schema changes detected. Please run the three graphql:generate configurations ('data', 'metadata', 'admin') and commit the changes."
            echo ""
            echo "The following GraphQL schema changes were detected:"
            echo "==================================================="
            git diff -- packages/twenty-front/src/generated packages/twenty-front/src/generated-metadata packages/twenty-front/src/generated-admin
            echo "==================================================="
            echo ""
            HAS_ERRORS=true
          fi

          npx nx run twenty-client-sdk:generate-metadata-client

          if ! git diff --quiet -- packages/twenty-client-sdk/src/metadata/generated; then
            echo "::error::SDK metadata client changes detected. Please run 'npx nx run twenty-client-sdk:generate-metadata-client' and commit the changes."
            echo ""
            echo "The following SDK metadata client changes were detected:"
            echo "==================================================="
            git diff -- packages/twenty-client-sdk/src/metadata/generated
            echo "==================================================="
            echo ""
            HAS_ERRORS=true
          fi

          if [ "$HAS_ERRORS" = true ]; then
            exit 1
          fi
  server-test:
    needs: server-build
    timeout-minutes: 30
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - name: Fetch custom Github Actions and base branch history
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Install dependencies
        uses: ./.github/actions/yarn-install
      - name: Restore server build cache
        uses: ./.github/actions/restore-cache
        with:
          key: ${{ env.SERVER_BUILD_CACHE_KEY }}
      - name: Build twenty-shared
        run: npx nx build twenty-shared
      - name: Server / Run Tests (shard ${{ matrix.shard }}/4)
        run: npx nx run twenty-server:test:ci --shard=${{ matrix.shard }}/4

  server-integration-test:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    needs: server-build
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
    services:
      postgres:
        image: postgres:18
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis
        ports:
          - 6379:6379
      clickhouse:
        image: clickhouse/clickhouse-server:25.8.8
        env:
          CLICKHOUSE_PASSWORD: clickhousePassword
          CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty"
        ports:
          - 8123:8123
          - 9000:9000
        options: >-
          --health-cmd "clickhouse-client --host=localhost --port=9000 --user=default --password=clickhousePassword --query='SELECT 1'"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    env:
      NODE_ENV: test
      ANALYTICS_ENABLED: true
      CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty"
      CLICKHOUSE_PASSWORD: clickhousePassword
      SHARD_COUNTER: 16
    steps:
      - name: Fetch custom Github Actions and base branch history
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Install dependencies
        uses: ./.github/actions/yarn-install
      - name: Update .env.test for integrations tests
        run: |
          echo "" >> .env.test
          echo "IS_BILLING_ENABLED=true" >> .env.test
          echo "BILLING_STRIPE_API_KEY=test-api-key" >> .env.test
          echo "BILLING_STRIPE_BASE_PLAN_PRODUCT_ID=test-base-plan-product-id" >> .env.test
          echo "BILLING_STRIPE_WEBHOOK_SECRET=test-webhook-secret" >> .env.test
          echo "BILLING_PLAN_REQUIRED_LINK=http://localhost:3001/stripe-redirection" >> .env.test
      - name: Restore server build cache
        uses: ./.github/actions/restore-cache
        with:
          key: ${{ env.SERVER_BUILD_CACHE_KEY }}
      - name: Server / Build
        run: npx nx build twenty-server
      - name: Build dependencies
        run: |
          npx nx build twenty-shared
          npx nx build twenty-emails
      - name: Server / Create Test DB
        run: |
          PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
      - name: Run ClickHouse migrations
        run: npx nx clickhouse:migrate twenty-server
      - name: Run ClickHouse seeds
        run: npx nx clickhouse:seed twenty-server
      - name: Server / Run Integration Tests
        uses: ./.github/actions/nx-affected
        with:
          tag: scope:backend
          tasks: 'test:integration'
          configuration: 'with-db-reset'
          args: --shard=${{ matrix.shard }}/${{ env.SHARD_COUNTER }}

  cross-version-upgrade:
    needs: [upgrade-changed-files-check, server-build]
    uses: ./.github/workflows/ci-cross-version-upgrade.yaml
    with:
      skip: ${{ needs.upgrade-changed-files-check.outputs.any_changed != 'true' }}

  discover-public-apps:
    needs: changed-files-check
    if: needs.changed-files-check.outputs.any_changed == 'true'
    uses: ./.github/workflows/discover-apps.yaml
    with:
      scope: public
      changed-only: false

  server-apps-install-smoke:
    needs: [discover-public-apps, server-build]
    timeout-minutes: 30
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        app: ${{ fromJSON(needs.discover-public-apps.outputs.matrix) }}
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          fetch-depth: 10
      - name: Spawn Twenty server from monorepo
        id: twenty
        uses: ./.github/actions/spawn-twenty-server
        with:
          source: local
          server-build-cache-key: ${{ env.SERVER_BUILD_CACHE_KEY }}
      - name: Install app against server
        uses: ./.github/actions/test-twenty-app
        with:
          api-url: ${{ steps.twenty.outputs.server-url }}
          api-key: ${{ steps.twenty.outputs.api-key }}
          app-path: ${{ matrix.app.path }}
          mode: installation-only

  ci-server-status-check:
    if: always() && !cancelled()
    timeout-minutes: 5
    runs-on: ubuntu-latest
    needs:
      [
        changed-files-check,
        server-build,
        server-lint-typecheck,
        server-previous-version-upgrade-mutation-guard,
        server-validation,
        server-test,
        server-integration-test,
        cross-version-upgrade,
        discover-public-apps,
        server-apps-install-smoke,
      ]
    steps:
      - name: Fail job if any needs failed
        if: contains(needs.*.result, 'failure')
        run: exit 1