diff --git a/src/cli.ts b/src/cli.ts index 369530d..0bfd75e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,7 +1,10 @@ import { createApp } from './app.ts'; +import { registerI18nCommand } from './commands/i18n.ts'; const program = createApp(); +registerI18nCommand(program); + program.parseAsync(process.argv).catch((err) => { console.error(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/src/commands/i18n.ts b/src/commands/i18n.ts new file mode 100644 index 0000000..ea66e4d --- /dev/null +++ b/src/commands/i18n.ts @@ -0,0 +1,317 @@ +import type { Command } from 'commander'; +import fs from 'fs-extra'; +import path from 'node:path'; +import { getConfig } from '../config.ts'; +import { PUP_VERSION } from '../app.ts'; +import * as output from '../utils/output.ts'; +import type { I18nResolvedConfig } from '../types.ts'; + +/** + * Backoff multipliers for HTTP 429 rate limit errors. + * Index corresponds to the 429 occurrence count (0-indexed). + * Applied as: delay * multiplier. + * + * @since TBD + */ +const HTTP_429_BACKOFF_MULTIPLIERS = [16, 31, 91, 151]; + +/** + * Registers the `i18n` command with the CLI program. + * + * @since TBD + * + * @param {Command} program - The Commander.js program instance. + * + * @returns {void} + */ +export function registerI18nCommand(program: Command): void { + program + .command('i18n') + .description('Fetches language files for the project.') + .option('--retries ', 'How many retries per translation file.', '3') + .option('--delay ', 'Delay (seconds) between retries and for 429 backoff.', '2') + .option('--batch-size ', 'Batch size for grouping downloads.', '3') + .option('--root ', 'Set the root directory for downloading language files.') + .action(async (options: { retries?: string; delay?: string; batchSize?: string; root?: string }) => { + const config = getConfig(); + const i18nConfigs = config.getI18n(); + const cwd = options.root ?? config.getWorkingDir(); + const maxRetries = Math.max(1, Math.min(5, parseInt(options.retries ?? '3', 10))); + const delay = Math.max(1, parseInt(options.delay ?? '2', 10)); + const batchSize = Math.max(1, parseInt(options.batchSize ?? '3', 10)); + + if (i18nConfigs.length === 0) { + output.log('No i18n configuration found. Skipping.'); + return; + } + + for (const i18nConfig of i18nConfigs) { + const result = await downloadLanguageFiles(i18nConfig, cwd, maxRetries, delay, batchSize); + + if (result !== 0) { + output.error('Failed to download language files.'); + output.warning('Config:'); + output.log(JSON.stringify(i18nConfig, null, 2)); + process.exitCode = result; + return; + } + } + }); +} + +/** + * Extracts the wait time from the Retry-After header if present. + * Respects the server hint but caps it to the backoff schedule for that attempt. + * + * @since TBD + * + * @param {Response} response - The HTTP response containing potential Retry-After header. + * @param {number} backoffWait - The computed backoff wait time in seconds. + * + * @returns {number} The wait time in seconds. + */ +function getWaitTimeFor429(response: Response, backoffWait: number): number { + const retryAfter = response.headers.get('Retry-After'); + + if (!retryAfter) { + return backoffWait; + } + + // Retry-After can be numeric seconds or an HTTP-date; parse numeric only. + if (/^\d+$/.test(retryAfter)) { + const serverWait = parseInt(retryAfter, 10); + // Use the server hint but cap at our backoff (don't wait longer than we're willing to). + return Math.max(1, Math.min(serverWait, backoffWait)); + } + + // HTTP-date format is complex to parse; fall back to backoff schedule. + return backoffWait; +} + +/** + * Downloads language files for a single i18n configuration. + * Processes downloads sequentially with deterministic retry logic. + * + * @since TBD + * + * @param {I18nResolvedConfig} config - The resolved i18n configuration for this translation source. + * @param {string} cwd - The current working directory. + * @param {number} maxRetries - The maximum number of retry attempts for failed downloads. + * @param {number} delay - The base delay in seconds between retries and for 429 backoff. + * @param {number} batchSize - The batch size for grouping downloads. + * + * @returns {Promise} 0 on success, 1 on failure. + */ +async function downloadLanguageFiles( + config: I18nResolvedConfig, + cwd: string, + maxRetries: number, + delay: number, + batchSize: number +): Promise { + const projectUrl = config.url + .replace('{slug}', config.slug) + .replace('%slug%', config.slug); + + output.log(`Fetching language files for ${config.textdomain} from ${projectUrl}`); + + let data: TranslationApiResponse; + + try { + const response = await fetch(projectUrl, { + headers: { 'User-Agent': `StellarWP PUP/${PUP_VERSION}` }, + }); + if (!response.ok) { + output.error(`Failed to fetch project data from ${projectUrl}`); + return 1; + } + data = (await response.json()) as TranslationApiResponse; + } catch (err) { + output.error(`Failed to fetch translation data: ${err}`); + return 1; + } + + if ( + !data.translation_sets || + !Array.isArray(data.translation_sets) || + data.translation_sets.length === 0 + ) { + output.error(`Failed to fetch translation sets from ${projectUrl}`); + return 1; + } + + const minimumPercentage = config.filter.minimum_percentage; + + const langDir = path.resolve(cwd, config.path); + await fs.mkdirp(langDir); + + // Build a list of (translation, format) pairs to download. + const downloadItems: [TranslationSet, string][] = []; + + for (const translation of data.translation_sets) { + // Skip when translations are zero. + if (translation.current_count === 0) { + continue; + } + + // Skip any translation set that doesn't match the minimum percentage. + if (minimumPercentage > translation.percent_translated) { + continue; + } + + for (const format of config.formats) { + downloadItems.push([translation, format]); + } + } + + if (downloadItems.length === 0) { + return 0; + } + + // Process downloads sequentially in batches (for grouping/visibility). + let failedCount = 0; + + for (let offset = 0; offset < downloadItems.length; offset += batchSize) { + const batch = downloadItems.slice(offset, offset + batchSize); + + // Process each item in the batch sequentially. + for (const [translation, format] of batch) { + try { + await downloadAndSaveTranslationSync( + config, translation, format, projectUrl, langDir, maxRetries, delay + ); + } catch (err) { + output.error(`Download failed: ${err instanceof Error ? err.message : String(err)}`); + failedCount++; + } + } + } + + return failedCount > 0 ? 1 : 0; +} + +/** + * Synchronously downloads and saves a translation with retry logic. + * Retries consume the standard retry budget; 429 responses use smarter delay logic. + * + * @since TBD + * + * @param {I18nResolvedConfig} config - The resolved i18n configuration. + * @param {TranslationSet} translation - The translation set to download. + * @param {string} format - The file format to download (e.g. "po", "mo"). + * @param {string} projectUrl - The base project API URL. + * @param {string} langDir - The absolute path to the language files directory. + * @param {number} maxRetries - The maximum number of retry attempts. + * @param {number} delay - The base delay in seconds between retries and for 429 backoff. + * + * @returns {Promise} + */ +async function downloadAndSaveTranslationSync( + config: I18nResolvedConfig, + translation: TranslationSet, + format: string, + projectUrl: string, + langDir: string, + maxRetries: number, + delay: number +): Promise { + const translationUrl = `${projectUrl}/${translation.locale}/${translation.slug}/export-translations?format=${format}`; + let http429Count = 0; + + for (let tried = 0; tried < maxRetries; tried++) { + const response = await fetch(translationUrl, { + headers: { 'User-Agent': `StellarWP PUP/${PUP_VERSION}` }, + }); + const statusCode = response.status; + const buffer = Buffer.from(await response.arrayBuffer()); + const bodySize = buffer.byteLength; + + // Handle HTTP 429 (Too Many Requests) with smarter delay. + if (statusCode === 429) { + const multiplier = HTTP_429_BACKOFF_MULTIPLIERS[http429Count] ?? + HTTP_429_BACKOFF_MULTIPLIERS[HTTP_429_BACKOFF_MULTIPLIERS.length - 1]; + const backoffWait = delay * multiplier; + const waitTime = getWaitTimeFor429(response, backoffWait); + + output.warning( + `Rate limited (HTTP 429) on ${translation.slug}. Waiting ${waitTime}s before retry...` + ); + + await new Promise(resolve => setTimeout(resolve, waitTime * 1000)); + http429Count++; + continue; + } + + // Check for valid response (non-429 case). + if (statusCode !== 200 || bodySize < 200) { + // Non-429 failure: use standard delay and retry. + if (tried < maxRetries - 1) { + output.error( + `Invalid response from ${translationUrl} (status: ${statusCode}, size: ${bodySize}). Retrying...` + ); + await new Promise(resolve => setTimeout(resolve, delay * 1000)); + continue; + } + break; + } + + // Success: save and return. + saveTranslationFile(buffer, config, translation, format, langDir); + return; + } + + // All retries exhausted. + throw new Error(`Failed to download ${translation.slug} after ${maxRetries} retries`); +} + +/** + * Saves a translation file to disk. + * + * @since TBD + * + * @param {Buffer} content - The translation file content. + * @param {I18nResolvedConfig} config - The resolved i18n configuration. + * @param {TranslationSet} translation - The translation set metadata. + * @param {string} format - The file format (e.g. "po", "mo"). + * @param {string} langDir - The absolute path to the language files directory. + * + * @returns {void} + */ +function saveTranslationFile( + content: Buffer, + config: I18nResolvedConfig, + translation: TranslationSet, + format: string, + langDir: string +): void { + const filename = config.file_format + .replace('%domainPath%', config.path) + .replace('%textdomain%', config.textdomain) + .replace('%locale%', translation.locale ?? '') + .replace('%wp_locale%', translation.wp_locale ?? '') + .replace('%format%', format); + + const filePath = path.join(langDir, filename); + fs.writeFileSync(filePath, content); + + // Verify the written file size matches the response size. + const stat = fs.statSync(filePath); + if (stat.size !== content.byteLength) { + fs.unlinkSync(filePath); + throw new Error(`Failed to write translation to ${filePath}`); + } + + output.log(`* Translation created for ${filePath}`); +} + +interface TranslationApiResponse { + translation_sets: TranslationSet[]; +} + +interface TranslationSet { + locale: string; + wp_locale?: string; + slug: string; + current_count: number; + percent_translated: number; +} diff --git a/src/models/i18n-config.ts b/src/models/i18n-config.ts new file mode 100644 index 0000000..fc67ea5 --- /dev/null +++ b/src/models/i18n-config.ts @@ -0,0 +1,14 @@ +import type { I18nResolvedConfig } from '../types.js'; + +/** + * Creates a resolved i18n configuration object. + * + * @since TBD + * + * @param {I18nResolvedConfig} config - The i18n configuration to clone. + * + * @returns {I18nResolvedConfig} A new copy of the i18n configuration object. + */ +export function createI18nConfig(config: I18nResolvedConfig): I18nResolvedConfig { + return { ...config }; +} diff --git a/tests/commands/i18n.test.ts b/tests/commands/i18n.test.ts new file mode 100644 index 0000000..dcb6d6f --- /dev/null +++ b/tests/commands/i18n.test.ts @@ -0,0 +1,421 @@ +import http from 'node:http'; +import path from 'node:path'; +import fs from 'fs-extra'; +import { + runPup, + writePuprc, + getPuprc, + createTempProject, + cleanupTempProjects, +} from '../helpers/setup.js'; + +// A response body >= 200 bytes so it passes the minimum size check. +const fakeTranslationBody = Buffer.alloc(250, 0x78); + +describe('i18n command', () => { + let server: http.Server; + let serverUrl: string; + let projectDir: string; + const requestCounts: Record = {}; + + // Routes for the mock GlotPress API + const routes: Record = { + '/api/projects/wp-plugins/fake-project/stable': { + status: 200, + body: JSON.stringify({ + translation_sets: [ + { locale: 'de', wp_locale: 'de_DE', slug: 'default', current_count: 100, percent_translated: 80 }, + { locale: 'fr', wp_locale: 'fr_FR', slug: 'default', current_count: 50, percent_translated: 50 }, + { locale: 'ja', wp_locale: 'ja', slug: 'default', current_count: 5, percent_translated: 10 }, + ], + }), + contentType: 'application/json', + }, + '/api/projects/wp-plugins/fake-project/with-zero-count': { + status: 200, + body: JSON.stringify({ + translation_sets: [ + { locale: 'de', wp_locale: 'de_DE', slug: 'default', current_count: 100, percent_translated: 80 }, + { locale: 'es', wp_locale: 'es_ES', slug: 'default', current_count: 0, percent_translated: 90 }, + ], + }), + contentType: 'application/json', + }, + '/api/projects/wp-plugins/fake-project/rate-limited': { + status: 200, + body: JSON.stringify({ + translation_sets: [ + { locale: 'de', wp_locale: 'de_DE', slug: 'default', current_count: 100, percent_translated: 80 }, + ], + }), + contentType: 'application/json', + }, + '/api/projects/wp-plugins/fake-project/always-rate-limited': { + status: 200, + body: JSON.stringify({ + translation_sets: [ + { locale: 'de', wp_locale: 'de_DE', slug: 'default', current_count: 100, percent_translated: 80 }, + ], + }), + contentType: 'application/json', + }, + '/not-found': { + status: 404, + body: 'Not Found', + }, + '/bad-json': { + status: 200, + body: 'this is not json', + contentType: 'text/plain', + }, + '/no-sets': { + status: 200, + body: JSON.stringify({ something_else: true }), + contentType: 'application/json', + }, + }; + + beforeAll((done) => { + server = http.createServer((req, res) => { + const url = req.url ?? ''; + + // Track request counts per URL. + requestCounts[url] = (requestCounts[url] ?? 0) + 1; + + // Check for translation file download requests (contain "export-translations") + if (url.includes('export-translations')) { + // Always return 429 for the always-rate-limited project. + if (url.includes('always-rate-limited')) { + res.writeHead(429, { + 'Content-Type': 'text/plain', + 'Retry-After': '1', + }); + res.end('Too Many Requests'); + return; + } + + // Return 429 on first request for the rate-limited project, then 200. + if (url.includes('rate-limited') && requestCounts[url] === 1) { + res.writeHead(429, { + 'Content-Type': 'text/plain', + 'Retry-After': '1', + }); + res.end('Too Many Requests'); + return; + } + + res.writeHead(200, { 'Content-Type': 'application/octet-stream' }); + res.end(fakeTranslationBody); + return; + } + + const route = routes[url]; + if (route) { + res.writeHead(route.status, { 'Content-Type': route.contentType ?? 'text/plain' }); + res.end(route.body); + } else { + res.writeHead(404); + res.end('Unknown route'); + } + }); + + server.listen(0, '127.0.0.1', () => { + const addr = server.address(); + if (addr && typeof addr === 'object') { + serverUrl = `http://127.0.0.1:${addr.port}`; + } + done(); + }); + }); + + afterAll((done) => { + server.close(done); + }); + + beforeEach(() => { + projectDir = createTempProject(); + for (const key of Object.keys(requestCounts)) { + delete requestCounts[key]; + } + }); + + afterEach(() => { + cleanupTempProjects(); + }); + + it('should skip when no i18n config is set', async () => { + const puprc = getPuprc(); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('No i18n configuration found. Skipping.'); + }); + + it('should skip when i18n is an empty array', async () => { + const puprc = getPuprc({ i18n: [] }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('No i18n configuration found. Skipping.'); + }); + + it('should skip i18n entries missing required fields', async () => { + const puprc = getPuprc({ + i18n: [{ path: 'lang' }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('No i18n configuration found. Skipping.'); + }); + + it('should accept a single i18n object instead of an array', async () => { + const puprc = getPuprc({ + i18n: { + url: `${serverUrl}/api/projects/wp-plugins/%slug%/stable`, + textdomain: 'fake-project', + slug: 'fake-project', + }, + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('Fetching language files for'); + }); + + it('should handle non-200 HTTP response gracefully', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/not-found`, + textdomain: 'fake-project', + slug: 'fake-project', + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Failed to fetch project data from'); + }); + + it('should handle invalid JSON response gracefully', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/bad-json`, + textdomain: 'fake-project', + slug: 'fake-project', + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Failed to fetch translation data'); + }); + + it('should handle response missing translation_sets', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/no-sets`, + textdomain: 'fake-project', + slug: 'fake-project', + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Failed to fetch translation sets from'); + }); + + it('should fetch and download translations meeting threshold', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/stable`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'lang', + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('Fetching language files for fake-project from'); + + // Verify files were downloaded + const langDir = path.join(projectDir, 'lang'); + expect(fs.existsSync(langDir)).toBe(true); + + const files = fs.readdirSync(langDir); + // Default threshold is 30%, so de (80%) and fr (50%) should pass, ja (10%) should not + expect(files).toContain('fake-project-de_DE.po'); + expect(files).toContain('fake-project-de_DE.mo'); + expect(files).toContain('fake-project-fr_FR.po'); + expect(files).toContain('fake-project-fr_FR.mo'); + // ja should be excluded (10% < 30% threshold) + expect(files).not.toContain('fake-project-ja.po'); + }); + + it('should respect custom minimum_percentage filter', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/stable`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'lang', + filter: { minimum_percentage: 70 }, + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + + // Only de (80%) meets 70% threshold + const langDir = path.join(projectDir, 'lang'); + const files = fs.readdirSync(langDir); + expect(files).toContain('fake-project-de_DE.po'); + expect(files).toContain('fake-project-de_DE.mo'); + expect(files).not.toContain('fake-project-fr_FR.po'); + expect(files).not.toContain('fake-project-ja.po'); + }); + + it('should skip translations with zero current_count', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/with-zero-count`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'lang', + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + + const langDir = path.join(projectDir, 'lang'); + const files = fs.readdirSync(langDir); + // de has current_count: 100 and 80% translated, should be downloaded + expect(files).toContain('fake-project-de_DE.po'); + expect(files).toContain('fake-project-de_DE.mo'); + // es has current_count: 0, should be skipped even though 90% translated + expect(files).not.toContain('fake-project-es_ES.po'); + expect(files).not.toContain('fake-project-es_ES.mo'); + }); + + it('should clamp --retries to minimum of 1', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/stable`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'lang', + }], + }); + writePuprc(puprc, projectDir); + + // --retries 0 is clamped to 1, so downloads should succeed + const result = await runPup('i18n --retries 0', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + + const langDir = path.join(projectDir, 'lang'); + const files = fs.readdirSync(langDir); + expect(files).toContain('fake-project-de_DE.po'); + expect(files).toContain('fake-project-de_DE.mo'); + }); + + it('should use --root to download files to the specified directory', async () => { + // rootDir is an empty subdirectory — no .puprc, no project files. + // This proves config is loaded from projectDir (the cwd) and --root only + // controls where downloaded files are saved. + const rootDir = path.join(projectDir, 'alt-root'); + fs.mkdirpSync(rootDir); + + writePuprc(getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/stable`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'custom/translations', + }], + }), projectDir); + + const result = await runPup(`i18n --root ${rootDir}`, { cwd: projectDir }); + expect(result.exitCode).toBe(0); + + // Files should be in rootDir/custom/translations + const rootLangDir = path.join(rootDir, 'custom', 'translations'); + expect(fs.existsSync(rootLangDir)).toBe(true); + const files = fs.readdirSync(rootLangDir); + expect(files).toContain('fake-project-de_DE.po'); + expect(files).toContain('fake-project-de_DE.mo'); + + // And NOT directly in projectDir/custom/translations + expect(fs.existsSync(path.join(projectDir, 'custom'))).toBe(false); + }); + + it('should handle unreachable URL gracefully', async () => { + const puprc = getPuprc({ + i18n: [{ + url: 'http://127.0.0.1:1/does-not-exist', + textdomain: 'fake-project', + slug: 'fake-project', + }], + }); + writePuprc(puprc, projectDir); + + const result = await runPup('i18n', { cwd: projectDir }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Fetching language files for'); + expect(result.output).toContain('Failed to fetch translation data'); + }); + + it('should recover from HTTP 429 and download successfully', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/rate-limited`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'lang', + }], + }); + writePuprc(puprc, projectDir); + + // Server returns 429 on first translation request, then 200 on retry. + // Retry-After: 1 is capped to min(1, backoffWait), so effective wait = 1s. + const result = await runPup('i18n --retries 2 --delay 1', { cwd: projectDir }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('Rate limited (HTTP 429)'); + + // Files should still be downloaded after recovery + const langDir = path.join(projectDir, 'lang'); + const files = fs.readdirSync(langDir); + expect(files).toContain('fake-project-de_DE.po'); + expect(files).toContain('fake-project-de_DE.mo'); + }, 30000); + + it('should fail after exhausting retries on persistent 429', async () => { + const puprc = getPuprc({ + i18n: [{ + url: `${serverUrl}/api/projects/wp-plugins/%slug%/always-rate-limited`, + textdomain: 'fake-project', + slug: 'fake-project', + path: 'lang', + }], + }); + writePuprc(puprc, projectDir); + + // With --retries 1 --delay 1, it tries once, gets 429, waits, then exhausts retries. + const result = await runPup('i18n --retries 1 --delay 1', { cwd: projectDir }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Rate limited (HTTP 429)'); + expect(result.output).toContain('Download failed'); + expect(result.output).toContain('Failed to download language files'); + }, 30000); +});