From ce48ddc4770f2dd5d4c5ebd2aec96a4617e53d43 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sat, 14 Mar 2026 16:06:10 -0400 Subject: [PATCH] chore: migrate crowdin-dist to dedicated i18n repo --- .github/scripts/sync-crowdin-distribution.js | 194 ------------------ .../workflows/crowdin-distribution-sync.yml | 88 -------- src/js/crowdin.js | 2 +- tests/crowdin.test.js | 2 +- 4 files changed, 2 insertions(+), 284 deletions(-) delete mode 100644 .github/scripts/sync-crowdin-distribution.js delete mode 100644 .github/workflows/crowdin-distribution-sync.yml diff --git a/.github/scripts/sync-crowdin-distribution.js b/.github/scripts/sync-crowdin-distribution.js deleted file mode 100644 index 0eef0f8..0000000 --- a/.github/scripts/sync-crowdin-distribution.js +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env node -/** - * Syncs Crowdin distribution files from distributions.crowdin.net to a local directory. - * Designed to be run from GitHub Actions and produce a static-file artifact for GitHub Pages. - * - * Usage: - * node sync-crowdin-distribution.js - * - * Environment variables: - * OUTPUT_DIR - Directory to write files into (default: dist-pages/crowdin-dist) - */ - -'use strict'; - -const https = require('node:https'); -const fs = require('node:fs'); -const path = require('node:path'); -const zlib = require('node:zlib'); -const { promisify } = require('node:util'); - -const gunzip = promisify(zlib.gunzip); - -const BASE_CDN = 'https://distributions.crowdin.net'; -const OUTPUT_DIR = path.resolve(process.env.OUTPUT_DIR || 'dist-pages/crowdin-dist'); - -/** Number of simultaneous downloads per batch. */ -const CONCURRENCY = 8; - -/** - * Distribution hashes to sync. - * Read from the CROWDIN_DISTRIBUTION_IDS environment variable as a - * comma-separated list (e.g. "hash1,hash2"). Store the value in GitHub - * project variables under the name CROWDIN_DISTRIBUTION_IDS. - */ -const DISTRIBUTIONS = (process.env.CROWDIN_DISTRIBUTION_IDS || '') - .split(',') - .map((s) => s.trim()) - .filter(Boolean); - -if (DISTRIBUTIONS.length === 0) { - console.error('ERROR: CROWDIN_DISTRIBUTION_IDS environment variable is not set or empty.'); - process.exit(1); -} - -/** - * Collects all data chunks from an HTTP response stream into a single Buffer. - * @param {import('http').IncomingMessage} res - * @returns {Promise} - */ -function collectBody(res) { - return new Promise((resolve, reject) => { - const chunks = []; - res.on('data', (chunk) => chunks.push(chunk)); - res.on('end', () => resolve(Buffer.concat(chunks))); - res.on('error', reject); - }); -} - -/** - * Fetches a URL, following redirects, and returns the body as a Buffer. - * Transparently decompresses gzip-encoded responses so callers always receive - * plain bytes (the Crowdin CDN stores content files with Content-Encoding: gzip, - * but jsDelivr re-serves the raw bytes without that header). - * @param {string} url - * @returns {Promise} - */ -async function fetchUrl(url) { - return new Promise((resolve, reject) => { - https.get(url, async (res) => { - if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - return fetchUrl(res.headers.location).then(resolve, reject); - } - if (res.statusCode >= 400) { - return reject(new Error(`HTTP ${res.statusCode} for ${url}`)); - } - try { - const buffer = await collectBody(res); - const isGzip = res.headers['content-encoding'] === 'gzip'; - resolve(isGzip ? await gunzip(buffer) : buffer); - } catch (err) { - reject(err); - } - }).on('error', reject); - }); -} - -/** - * Writes data to a file, creating parent directories as needed. - * @param {string} filePath - * @param {Buffer|string} data - */ -function saveFile(filePath, data) { - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, data); -} - -/** - * Processes an array of items in fixed-size concurrent batches. - * @template T - * @param {T[]} items - * @param {number} batchSize - * @param {(item: T) => Promise} fn - */ -async function processInBatches(items, batchSize, fn) { - for (let i = 0; i < items.length; i += batchSize) { - await Promise.all(items.slice(i, i + batchSize).map(fn)); - } -} - -/** - * Downloads all distribution files for a single hash. - * @param {string} hash Distribution hash. - * @returns {Promise} true if all files were fetched without errors. - */ -async function syncDistribution(hash) { - console.log(`\n=== Syncing distribution: ${hash} ===`); - const hashDir = path.join(OUTPUT_DIR, hash); - - // manifest.json - console.log(' Fetching manifest.json...'); - const manifestBuf = await fetchUrl(`${BASE_CDN}/${hash}/manifest.json`); - saveFile(path.join(hashDir, 'manifest.json'), manifestBuf); - const manifest = JSON.parse(manifestBuf.toString('utf8')); - - console.log(` Timestamp : ${manifest.timestamp}`); - console.log(` Languages : ${(manifest.languages || []).length}`); - - // languages.json - console.log(' Fetching languages.json...'); - const langsBuf = await fetchUrl(`${BASE_CDN}/${hash}/languages.json`); - saveFile(path.join(hashDir, 'languages.json'), langsBuf); - - // content files - const contentPaths = new Set(); - if (manifest.content) { - for (const paths of Object.values(manifest.content)) { - for (const p of paths) { - contentPaths.add(p); - } - } - } - - const pathList = [...contentPaths]; - console.log(` Content files: ${pathList.length} (concurrency=${CONCURRENCY})`); - - let fetched = 0; - let failed = 0; - - await processInBatches(pathList, CONCURRENCY, async (contentPath) => { - const url = `${BASE_CDN}/${hash}${contentPath}`; - const localPath = path.join(hashDir, contentPath); - try { - const data = await fetchUrl(url); - saveFile(localPath, data); - fetched++; - if ((fetched + failed) % 50 === 0) { - console.log(` Progress: ${fetched + failed}/${pathList.length}`); - } - } catch (err) { - failed++; - console.warn(` WARN: failed to fetch ${contentPath}: ${err.message}`); - } - }); - - console.log(` Result: ${fetched} fetched, ${failed} failed`); - return failed === 0; -} - -async function main() { - console.log('Crowdin Distribution Sync'); - console.log(`Output dir: ${OUTPUT_DIR}`); - console.log(`Distributions: ${DISTRIBUTIONS.length}`); - - fs.mkdirSync(OUTPUT_DIR, { recursive: true }); - - let allOk = true; - for (const hash of DISTRIBUTIONS) { - try { - const ok = await syncDistribution(hash); - if (!ok) allOk = false; - } catch (err) { - console.error(`\nFATAL: Failed to sync ${hash}:`, err.message); - allOk = false; - } - } - - if (!allOk) { - console.error('\nSync completed with errors.'); - process.exit(1); - } - console.log('\nSync complete!'); -} - -main(); diff --git a/.github/workflows/crowdin-distribution-sync.yml b/.github/workflows/crowdin-distribution-sync.yml deleted file mode 100644 index 7c4d2ce..0000000 --- a/.github/workflows/crowdin-distribution-sync.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# Syncs Crowdin distribution files from distributions.crowdin.net to a -# dedicated git branch (crowdin-dist) served via jsDelivr CDN. -# -# proxy-translator.js fetches manifest.json, languages.json and all translation -# JSON files from https://distributions.crowdin.net. Those requests count -# against the LizardByte Crowdin free-tier quota, so we mirror the content -# here (refreshed daily) and redirect browser fetch() calls to jsDelivr via -# the interceptor in src/js/crowdin.js. -# -# jsDelivr CDN URL pattern: -# https://cdn.jsdelivr.net/gh/LizardByte/shared-web@crowdin-dist//… -# -# jsDelivr guarantees Access-Control-Allow-Origin: * on all responses, which -# means no CORS plugin is required in consumer pages. - -name: Sync Crowdin Distribution -permissions: {} - -on: - schedule: - # Run daily at 02:00 UTC so translations are fresh at the start of each day. - - cron: '0 2 * * *' - workflow_dispatch: # Allow ad-hoc manual runs - -# Only one deployment at a time; do not cancel an in-progress run. -concurrency: - group: crowdin-dist-sync - cancel-in-progress: false - -jobs: - sync: - name: Sync distributions to crowdin-dist branch - runs-on: ubuntu-latest - permissions: - contents: write - environment: - name: crowdin-dist - url: ${{ github.server_url }}/${{ github.repository }}/tree/crowdin-dist - if: github.repository_owner == 'LizardByte' # don't run for forks - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - token: ${{ secrets.GH_BOT_TOKEN }} - - - name: Set up Node.js - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: 'node' - - - name: Download Crowdin distribution files - env: - CROWDIN_DISTRIBUTION_IDS: ${{ vars.CROWDIN_DISTRIBUTION_IDS }} - OUTPUT_DIR: /tmp/crowdin-dist - run: node .github/scripts/sync-crowdin-distribution.js - - - name: Commit and push to crowdin-dist branch - env: - GH_BOT_NAME: ${{ vars.GH_BOT_NAME }} - GH_BOT_EMAIL: ${{ secrets.GH_BOT_EMAIL }} - run: | - git config user.name "${GH_BOT_NAME}" - git config user.email "${GH_BOT_EMAIL}" - - # Create an orphan branch so the branch contains only distribution - # files with no history from main (keeps the branch lean). - git checkout --orphan crowdin-dist - - # Remove every file that was inherited from the main checkout. - git rm -rf . --quiet - - # Clean up any remaining untracked files / directories. - git clean -fdx - - # Populate the branch with the freshly downloaded distribution files. - cp -r /tmp/crowdin-dist/. . - - git add . - - # Only commit when there are actual changes. - if git diff --staged --quiet; then - echo "No changes – distribution files are already up to date." - else - git commit -m "chore: sync Crowdin distributions" - git push origin crowdin-dist --force - fi diff --git a/src/js/crowdin.js b/src/js/crowdin.js index 0cee09c..69fadc4 100644 --- a/src/js/crowdin.js +++ b/src/js/crowdin.js @@ -8,7 +8,7 @@ const loadScript = require('./load-script'); * Structure mirrors https://distributions.crowdin.net//… exactly. * @type {string} */ -const CROWDIN_DIST_MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/shared-web@crowdin-dist'; +const CROWDIN_DIST_MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/i18n@dist'; /** * Monkey-patches globalThis.fetch to redirect Crowdin distribution requests to diff --git a/tests/crowdin.test.js b/tests/crowdin.test.js index 97286a4..437e16d 100644 --- a/tests/crowdin.test.js +++ b/tests/crowdin.test.js @@ -121,7 +121,7 @@ describe('initCrowdIn', () => { describe('Crowdin fetch interceptor', () => { const CROWDIN_CDN = 'https://distributions.crowdin.net'; - const MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/shared-web@crowdin-dist'; + const MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/i18n@dist'; beforeEach(() => { // Reset interceptor state so each test gets a fresh install