From ba8e1bf5a36e01f2061576a29dd9e8198156cd1c Mon Sep 17 00:00:00 2001 From: Abdul Rahman <81605929+abdulrahmancodes@users.noreply.github.com> Date: Tue, 30 Jun 2026 04:40:55 +0530 Subject: [PATCH] =?UTF-8?q?chore(docs):=20self-clean=20orphans=20and=20sur?= =?UTF-8?q?face=20failed=20languages=20in=20i18n=20=E2=80=A6=20(#22278)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Two robustness fixes to `docs-i18n-pull.yaml` so localized docs can't silently drift: 1. **Prune orphan localized files.** The pull only adds/updates files, never deletes — so localized copies of renamed/moved/deleted English pages linger and serve dead URLs (recently ~113 of them). A new `prune-orphan-translations` script (run with `--apply` in the workflow, on real pulls only) removes any `l//**` file whose English source no longer exists. 2. **Surface per-language download failures.** The loop previously swallowed failures with `|| echo "Warning..."`, so a language whose Crowdin server-side build fails (e.g. `ja`, failing at 79%) was skipped *silently* and froze indefinitely while every other language updated. We now collect failures, still commit the languages that succeeded, and **fail the run at the end** so a broken language is visible. --------- Co-authored-by: Cursor --- .github/workflows/docs-i18n-pull.yaml | 22 +++- .github/workflows/docs-i18n-push.yaml | 1 + .github/workflows/i18n-push.yaml | 1 + .github/workflows/website-i18n-push.yaml | 1 + package.json | 1 + .../scripts/prune-orphan-translations.ts | 116 ++++++++++++++++++ 6 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 packages/twenty-docs/scripts/prune-orphan-translations.ts diff --git a/.github/workflows/docs-i18n-pull.yaml b/.github/workflows/docs-i18n-pull.yaml index bcb0146a7f..87a17e4da7 100644 --- a/.github/workflows/docs-i18n-pull.yaml +++ b/.github/workflows/docs-i18n-pull.yaml @@ -71,14 +71,17 @@ jobs: # Pull docs translations from Crowdin one language at a time # This avoids build timeout issues when processing all languages at once - name: Pull translated docs from Crowdin + id: pull if: github.event_name != 'pull_request' && (inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') run: | # Languages supported by Mintlify (see packages/twenty-docs/src/shared/supported-languages.ts) LANGUAGES="fr ar cs de es it ja ko pt ro ru tr zh-CN" + FAILED_LANGUAGES="" + for lang in $LANGUAGES; do echo "=== Pulling translations for $lang ===" - crowdin download \ + if ! crowdin download \ --config .github/crowdin-docs.yml \ --token "$CROWDIN_PERSONAL_TOKEN" \ --base-url "https://twenty.api.crowdin.com" \ @@ -86,11 +89,15 @@ jobs: --skip-untranslated-strings=false \ --skip-untranslated-files=false \ --export-only-approved=false \ - --verbose || echo "Warning: Failed to pull $lang, continuing with other languages..." + --verbose; then + echo "::warning::Crowdin download failed for $lang (likely a server-side build failure); its translations will be stale until fixed." + FAILED_LANGUAGES="$FAILED_LANGUAGES $lang" + fi echo "" done - echo "=== Download complete ===" + echo "failed_languages=${FAILED_LANGUAGES# }" >> "$GITHUB_OUTPUT" + echo "=== Download complete (failed:${FAILED_LANGUAGES:-none}) ===" env: CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} @@ -100,6 +107,9 @@ jobs: - name: Fix translated documentation links run: bash packages/twenty-docs/scripts/fix-translated-links.sh + - name: Prune orphan localized translations + if: github.event_name != 'pull_request' && (inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + run: yarn docs:prune-orphans --apply - name: Regenerate navigation template if: github.event_name == 'pull_request' @@ -168,3 +178,9 @@ jobs: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | gh workflow run automerge-i18n.yaml --repo twentyhq/twenty-infra --ref main + + - name: Fail run if any language failed to download + if: always() && steps.pull.outputs.failed_languages != '' + run: | + echo "::error::Crowdin download failed for:${{ steps.pull.outputs.failed_languages }}. These languages were skipped and are now stale — fix the source (e.g. build-breaking translations) and re-run." + exit 1 diff --git a/.github/workflows/docs-i18n-push.yaml b/.github/workflows/docs-i18n-push.yaml index 99565598e7..33685c3f36 100644 --- a/.github/workflows/docs-i18n-push.yaml +++ b/.github/workflows/docs-i18n-push.yaml @@ -39,6 +39,7 @@ jobs: uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2 with: upload_sources: true + upload_sources_args: '--delete-obsolete' upload_translations: false download_translations: false localization_branch_name: i18n-docs diff --git a/.github/workflows/i18n-push.yaml b/.github/workflows/i18n-push.yaml index ca0fd4d5fe..55de858638 100644 --- a/.github/workflows/i18n-push.yaml +++ b/.github/workflows/i18n-push.yaml @@ -83,6 +83,7 @@ jobs: uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2 with: upload_sources: true + upload_sources_args: '--delete-obsolete' upload_translations: true download_translations: false localization_branch_name: i18n diff --git a/.github/workflows/website-i18n-push.yaml b/.github/workflows/website-i18n-push.yaml index a308777354..fb2743504d 100644 --- a/.github/workflows/website-i18n-push.yaml +++ b/.github/workflows/website-i18n-push.yaml @@ -81,6 +81,7 @@ jobs: uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2 with: upload_sources: true + upload_sources_args: '--delete-obsolete' upload_translations: true download_translations: false localization_branch_name: i18n-website diff --git a/package.json b/package.json index 849bf9d4b8..711245477a 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "docs:generate": "tsx packages/twenty-docs/scripts/generate-docs-json.ts", "docs:generate-navigation-template": "tsx packages/twenty-docs/scripts/generate-navigation-template.ts", "docs:generate-paths": "tsx packages/twenty-docs/scripts/generate-documentation-paths.ts", + "docs:prune-orphans": "tsx packages/twenty-docs/scripts/prune-orphan-translations.ts", "start": "npx concurrently --kill-others 'npx nx run-many -t start -p twenty-server twenty-front' 'npx wait-on tcp:3000 && npx nx run twenty-server:worker'" }, "workspaces": { diff --git a/packages/twenty-docs/scripts/prune-orphan-translations.ts b/packages/twenty-docs/scripts/prune-orphan-translations.ts new file mode 100644 index 0000000000..175608661d --- /dev/null +++ b/packages/twenty-docs/scripts/prune-orphan-translations.ts @@ -0,0 +1,116 @@ +import fs from 'fs'; +import path from 'path'; + +// Removes orphan localized docs: files under packages/twenty-docs/l//** +// whose corresponding English source page no longer exists. The Crowdin i18n +// pull only ever adds/updates files (never deletes), so when an English page is +// renamed, moved, or removed, its localized copies linger and keep serving +// dead/stale URLs. This keeps the localized tree in sync with the source tree. +// +// DRY-RUN by default (prints what it would remove); pass --apply to delete. +// +// Usage: +// tsx packages/twenty-docs/scripts/prune-orphan-translations.ts +// tsx packages/twenty-docs/scripts/prune-orphan-translations.ts --apply + +const DOCS_ROOT = path.resolve(__dirname, '..'); +const LOCALIZED_DIR = path.join(DOCS_ROOT, 'l'); + +// Safety: never delete a language's localized files wholesale. If more than this +// fraction of a language's files look orphaned, something is wrong (e.g. the +// English source tree wasn't checked out) — abort instead of pruning. +const MAX_ORPHAN_RATIO_PER_LANGUAGE = 0.25; + +const apply = process.argv.slice(2).includes('--apply'); + +const walkMdxFiles = (dir: string): string[] => { + const result: string[] = []; + for (const entry of fs.readdirSync(dir)) { + const fullPath = path.join(dir, entry); + if (fs.statSync(fullPath).isDirectory()) { + result.push(...walkMdxFiles(fullPath)); + } else if (entry.endsWith('.mdx')) { + result.push(fullPath); + } + } + return result; +}; + +// English source path a localized file mirrors: strip the leading `l//`. +const sourcePathOf = (localizedFile: string): string => { + const relativeToL = path.relative(LOCALIZED_DIR, localizedFile); + const segments = relativeToL.split(path.sep); + // segments[0] is the language code; the rest is the source-relative path. + return path.join(DOCS_ROOT, ...segments.slice(1)); +}; + +const languageOf = (localizedFile: string): string => + path.relative(LOCALIZED_DIR, localizedFile).split(path.sep)[0]; + +const main = (): void => { + if (!fs.existsSync(LOCALIZED_DIR)) { + console.log('No localized docs directory (packages/twenty-docs/l) — nothing to prune.'); + return; + } + + // Sanity guard: if the English source tree is empty, the checkout is broken; + // refuse to prune so we never mass-delete translations by mistake. + const englishFileCount = walkMdxFiles(DOCS_ROOT).filter( + (file) => !file.startsWith(`${LOCALIZED_DIR}${path.sep}`), + ).length; + + if (englishFileCount === 0) { + throw new Error( + 'Refusing to prune: found 0 English source .mdx files (source tree missing?).', + ); + } + + const localizedFiles = walkMdxFiles(LOCALIZED_DIR); + const orphans = localizedFiles.filter( + (file) => !fs.existsSync(sourcePathOf(file)), + ); + + if (orphans.length === 0) { + console.log(`No orphan localized files (${localizedFiles.length} scanned).`); + return; + } + + // Per-language blast-radius check. + const filesByLanguage = new Map(); + const orphansByLanguage = new Map(); + for (const file of localizedFiles) { + const language = languageOf(file); + filesByLanguage.set(language, (filesByLanguage.get(language) ?? 0) + 1); + } + for (const file of orphans) { + const language = languageOf(file); + orphansByLanguage.set(language, (orphansByLanguage.get(language) ?? 0) + 1); + } + + for (const [language, orphanCount] of orphansByLanguage) { + const total = filesByLanguage.get(language) ?? 0; + const ratio = total === 0 ? 1 : orphanCount / total; + if (ratio > MAX_ORPHAN_RATIO_PER_LANGUAGE) { + throw new Error( + `Refusing to prune: ${orphanCount}/${total} (${Math.round(ratio * 100)}%) of "${language}" files look orphaned, ` + + `over the ${Math.round(MAX_ORPHAN_RATIO_PER_LANGUAGE * 100)}% safety cap. Check the source checkout before pruning.`, + ); + } + } + + console.log( + `${apply ? 'Removing' : 'Would remove'} ${orphans.length} orphan localized file(s) of ${localizedFiles.length} scanned:`, + ); + for (const file of orphans.sort()) { + console.log(` ${apply ? 'DELETE' : 'orphan'} ${path.relative(DOCS_ROOT, file)}`); + if (apply) { + fs.rmSync(file); + } + } + + console.log( + `${apply ? 'Removed' : 'Would remove'}: ${orphans.length} file(s) across ${orphansByLanguage.size} language(s).`, + ); +}; + +main();