Skip to content

fix(crowdin): preserve whitespace within translation pipeline #382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ReenigneArcher merged 2 commits into from
Nov 7, 2026
Merged

fix(crowdin): preserve whitespace within translation pipeline #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f4dfcd5
fix(crowdin): preserve whitespace within translation pipeline
ReenigneArcher Nov 7, 2026
40f58f5
docs: revert Crowdin whitespace README addition
ReenigneArcher Nov 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 53 additions & 181 deletions src/js/crowdin.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,198 +12,71 @@ const CROWDIN_DIST_MIRROR = 'https://cdn.jsdelivr.net/gh/LizardByte/i18n@dist';
const CROWDIN_CODE_BLOCK_CLASS = 'crowdin-ignore-code';
const CROWDIN_PLATFORM_STYLING_MAX_ATTEMPTS = 100;
const CROWDIN_PLATFORM_STYLING_RETRY_DELAY_MS = 50;
const CROWDIN_INLINE_ELEMENT_SELECTOR = [
'a',
'abbr',
'b',
'cite',
'code',
'del',
'em',
'i',
'ins',
'kbd',
'mark',
'q',
's',
'samp',
'small',
'span',
'strong',
'sub',
'sup',
'time',
'u',
'var',
].join(',');
const CROWDIN_WHITESPACE_POST_PROCESSOR = 'crowdin-preserve-whitespace';
const CROWDIN_WHITESPACE_RENDERERS = new WeakSet();

/**
* Returns a sibling when it is an inline element whose boundary Crowdin may rewrite.
* @param {Node|null} sibling Candidate sibling.
* @returns {Element|null} Matching inline element.
* Preserves source whitespace in translation results before i18nextify records
* or renders them. Its cleanWhitespace handling only recognizes single-line
* text; multiline missing keys and translations can lose their boundary spaces.
* @param {*} value Translated value.
* @param {string|string[]} key Translation lookup keys.
* @param {Object} options Translation options, including the source defaultValue.
* @returns {*} Translation with the source's exact boundary whitespace.
*/
function _getCrowdinInlineSibling(sibling) {
if (!(sibling instanceof globalThis.Element)) return null;
return sibling.matches(CROWDIN_INLINE_ELEMENT_SELECTOR) ? sibling : null;
function _preserveCrowdinWhitespace(value, key, options) {
const source = options?.defaultValue;
if (typeof value !== 'string' || typeof source !== 'string') return value;
if (source.trim() === '') return source;

const leadingWhitespace = source.slice(0, source.length - source.trimStart().length);
const trailingWhitespace = source.slice(source.trimEnd().length);
return leadingWhitespace + value.trim() + trailingWhitespace;
}

/**
* Returns the exact whitespace at the start of a string.
* @param {string} text Text to inspect.
* @returns {string} Leading whitespace.
* Adds a postprocessor to Crowdin's renderer before its initial translation.
* Keeping i18nextify's cached translation and DOM text identical avoids the
* mutation-observer loop caused by repairing whitespace after rendering.
*/
function _getLeadingWhitespace(text) {
const trimmedText = text.trimStart();
return text.slice(0, text.length - trimmedText.length);
}

/**
* Returns the exact whitespace at the end of a string.
* @param {string} text Text to inspect.
* @returns {string} Trailing whitespace.
*/
function _getTrailingWhitespace(text) {
const trimmedText = text.trimEnd();
return text.slice(trimmedText.length);
}

/**
* Records whitespace that separates text from inline elements before Crowdin translates the page.
* @returns {Array<{
* node: Text,
* leading: boolean,
* leadingWhitespace: string,
* trailing: boolean,
* trailingWhitespace: string,
* previousInline: Element|null,
* nextInline: Element|null,
* whitespaceOnly: boolean,
* lastRestoredNode: Text|null,
* lastRestoredFrom: string|null
* }>} Recorded text-node boundaries.
*/
function _captureCrowdinWhitespaceBoundaries() {
const boundaries = [];
const walker = document.createTreeWalker(document.body, globalThis.NodeFilter.SHOW_TEXT);
let node = walker.nextNode();

while (node !== null) {
const previousInline = _getCrowdinInlineSibling(node.previousSibling);
const nextInline = _getCrowdinInlineSibling(node.nextSibling);
const leadingWhitespace = previousInline === null ? '' : _getLeadingWhitespace(node.data);
const trailingWhitespace = nextInline === null ? '' : _getTrailingWhitespace(node.data);
const leading = leadingWhitespace !== '';
const trailing = trailingWhitespace !== '';

if (leading || trailing) {
boundaries.push({
node,
leading,
leadingWhitespace,
trailing,
trailingWhitespace,
previousInline,
nextInline,
whitespaceOnly: node.data.trim() === '',
lastRestoredNode: null,
lastRestoredFrom: null,
});
}

node = walker.nextNode();
}

return boundaries;
}

/**
* Finds the current text node for a boundary after Crowdin changes the DOM.
* @param {Object} boundary Recorded whitespace boundary.
* @returns {Text|null} Current or recreated text node.
*/
function _resolveCrowdinWhitespaceNode(boundary) {
if (boundary.node.isConnected) return boundary.node;

const previousReplacement = boundary.previousInline?.nextSibling;
if (previousReplacement instanceof globalThis.Text && previousReplacement.isConnected) {
boundary.node = previousReplacement;
return boundary.node;
}
const nextReplacement = boundary.nextInline?.previousSibling;
if (nextReplacement instanceof globalThis.Text && nextReplacement.isConnected) {
boundary.node = nextReplacement;
return boundary.node;
}
if (!boundary.whitespaceOnly) return null;

const replacement = document.createTextNode('');
if (boundary.previousInline?.isConnected) {
boundary.previousInline.after(replacement);
} else if (boundary.nextInline?.isConnected) {
boundary.nextInline.before(replacement);
} else {
return null;
}

boundary.node = replacement;
return boundary.node;
}

/**
* Restores whitespace that Crowdin removed from translated text-node boundaries.
* @param {Array<Object>} boundaries Recorded text-node boundaries.
*/
function _restoreCrowdinWhitespaceBoundaries(boundaries) {
boundaries.forEach((boundary) => {
const node = _resolveCrowdinWhitespaceNode(boundary);
if (node === null) return;
if (boundary.lastRestoredNode === node && boundary.lastRestoredFrom === node.data) return;

const originalText = node.data;

const currentLeadingWhitespace = _getLeadingWhitespace(node.data);
if (boundary.leading && currentLeadingWhitespace !== boundary.leadingWhitespace) {
node.data = boundary.leadingWhitespace + node.data.slice(currentLeadingWhitespace.length);
}
const currentTrailingWhitespace = _getTrailingWhitespace(node.data);
if (boundary.trailing && currentTrailingWhitespace !== boundary.trailingWhitespace) {
const translatedTextEnd = node.data.length - currentTrailingWhitespace.length;
node.data = node.data.slice(0, translatedTextEnd) + boundary.trailingWhitespace;
}
if (node.data !== originalText) {
// A translator may observe this correction and remove the same
// whitespace again. Do not repeat the correction indefinitely.
boundary.lastRestoredNode = node;
boundary.lastRestoredFrom = originalText;
}
function _installCrowdinWhitespacePreserver() {
const renderer = globalThis.i18nextify;
if (CROWDIN_WHITESPACE_RENDERERS.has(renderer)) return;

renderer.i18next.use({
type: 'postProcessor',
name: CROWDIN_WHITESPACE_POST_PROCESSOR,
process: _preserveCrowdinWhitespace,
});
}

/**
* Creates the Website Translator callback and observes later translation mutations.
* @param {Array<Object>} boundaries Recorded text-node boundaries.
* @returns {Function} Website Translator callback.
*/
function _createCrowdinTranslationCallback(boundaries) {
const translationObserver = new globalThis.MutationObserver(restoreAndObserve);

function restoreAndObserve() {
translationObserver.disconnect();
_restoreCrowdinWhitespaceBoundaries(boundaries);
translationObserver.observe(document.body, {
characterData: true,
childList: true,
subtree: true,
});
}

return restoreAndObserve;
const originalInit = renderer.init;
renderer.init = function(options, ...args) {
const postProcess = options.postProcess ?? [];
const rendererOptions = {
...options,
postProcess: [
...(Array.isArray(postProcess) ? postProcess : [postProcess]),
CROWDIN_WHITESPACE_POST_PROCESSOR,
],
};
// i18next calls this handler after postprocessing a missing key, so
// preserve the boundaries of fallback and instant translations too.
if (typeof options.parseMissingKeyHandler === 'function') {
const originalHandler = options.parseMissingKeyHandler;
rendererOptions.parseMissingKeyHandler = function(key, defaultValue, translationOptions) {
const value = originalHandler.call(this, key, defaultValue, translationOptions);
return _preserveCrowdinWhitespace(value, key, translationOptions);
};
}
return originalInit.call(this, rendererOptions, ...args);
};
CROWDIN_WHITESPACE_RENDERERS.add(renderer);
}

/**
* Keeps preformatted examples out of Crowdin's text-node rewriting. Syntax
* highlighters can replace their children after translation starts, leaving
* the whitespace boundary snapshots pointing at detached nodes.
* highlighters can replace their children after translation starts. Leave their
* markup and whitespace under the highlighter's control.
*/
function _excludeCrowdinCodeBlocks() {
document.querySelectorAll('pre, .fragment').forEach((block) => {
Expand Down Expand Up @@ -356,11 +229,10 @@ function initCrowdIn(project = 'LizardByte', platform = 'jekyll') {

// Initialize Crowdin translator
_excludeCrowdinCodeBlocks();
const whitespaceBoundaries = _captureCrowdinWhitespaceBoundaries();
_installCrowdinWhitespacePreserver();

globalThis.proxyTranslator.init({
baseUrl: currentBaseUrl,
callback: _createCrowdinTranslationCallback(whitespaceBoundaries),
crawlerIgnoreClasses: [CROWDIN_CODE_BLOCK_CLASS],
distribution: projectSettings[project].distribution,
defaultLanguage: "en",
Expand Down
Loading
Loading