Skip to content

Commit

Permalink
dev: Add duplication check in translation script
Browse files Browse the repository at this point in the history
  • Loading branch information
@kschiffer
kschiffer committed Oct 16, 2023
1 parent 1840a82 commit cc189e5
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 44 deletions.
2 changes: 1 addition & 1 deletion tools/mage/js.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ func (js Js) ExtractLocaleFiles() error {
if mg.Verbose() {
fmt.Println("Building frontend locale files")
}
return sh.Run("node", "tools/mage/translations.js")
return sh.Run("node", "tools/mage/translations.js", "--ignore-duplicates")
}

// BackendTranslations builds the backend locale files.
Expand Down
128 changes: 85 additions & 43 deletions tools/mage/translations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

/* global process */
/* eslint-disable no-alert, no-console */
/* eslint-disable import/no-commonjs */

const fs = require('fs')
const path = require('path')
Expand All @@ -32,7 +33,7 @@ const env = process.env
* @param {string|Array<string>} s - The string or array to flatten.
* @returns {Array<string>} - A flat array of strings.
*/
const flatten = function (s) {
const flatten = s => {
if (!s) {
return []
}
Expand All @@ -52,6 +53,8 @@ const backendMessages = argv.backendMessages || env.MESSAGES_BACKEND || 'config/
const defaultLocale = argv.default || env.DEFAULT_LOCALE || 'en'
const exportForTranslation = argv['export-for-interpreters'] || env.EXPORT_FOR_INTERPRETERS || false
const verbose = ('verbose' in argv && argv.verbose !== 'false') || false
const ignoreDuplicates =
('ignore-duplicates' in argv && argv['ignore-duplicates'] !== 'false') || false

if (argv.help) {
console.log(`Usage: translations [opts]
Expand All @@ -76,6 +79,7 @@ Options:
--backend-only <flag>. Flag that determines whether only backend messages will be processed
--verbose verbose output for debugging purposes
--export-for-translation Flag that determines whether the messages will be exported for interpreters
--ignore-duplicates Flag that determines whether the program will ignore duplicate messages
--help show this help message
`)
}
Expand All @@ -85,38 +89,36 @@ Options:
*
* @param {string} pat - The glob pattern, eg. "./foo/*.js".
* @returns {Promise<Array<string>>} - A promise that resolves to an array of
* filenames that match the pattern.
* filenames that match the pattern.
*/
const glob = function (pat) {
return new Promise(function (resolve, reject) {
g(pat, function (err, res) {
const glob = pat =>
new Promise((resolve, reject) => {
g(pat, (err, res) => {
if (err) {
return reject(err)
}
return resolve(res)
})
})
}

/**
* Read a file from disk.
*
* @param {string} filename - The name of the file to read.
* @returns {Promise<string>} - The contents of the file.
*/
const read = function (filename) {
return new Promise(function (resolve, reject) {
const read = filename =>
new Promise((resolve, reject) => {
if (verbose) {
console.log(`reading from ${filename}`)
}
fs.readFile(filename, function (err, res) {
fs.readFile(filename, (err, res) => {
if (err) {
return reject(err)
}
return resolve(res)
})
})
}

/**
* Write to a file.
Expand All @@ -126,19 +128,18 @@ const read = function (filename) {
*
* @returns {Promise<undefined>} - A promise that resolves when the file has been written.
*/
const write = function (filename, content) {
return new Promise(function (resolve, reject) {
const write = (filename, content) =>
new Promise((resolve, reject) => {
if (verbose) {
console.log('writing', filename)
}
fs.writeFile(filename, content, function (err, res) {
fs.writeFile(filename, content, (err, res) => {
if (err) {
return reject(err)
}
return resolve(res)
})
})
}

/**
* Read the locales from the localesDir (specified by --locales) and parse
Expand All @@ -147,24 +148,67 @@ const write = function (filename, content) {
* Locales that are in the localesDir but not in --support will be omitted.
*
* @returns {object} - The locales, keyed by locale name.
* For example: `{ en: { ... }, ja: { ... }}`.
* For example: `{ en: { ... }, ja: { ... }}`.
*/
const readLocales = async function () {
const readLocales = async () => {
const loc = await Promise.all(
support.map(async function (locale) {
support.map(async locale => {
let parsed = {}
let content
try {
const content = await read(`${path.resolve(localesDir)}/${locale}.json`)
content = await read(`${path.resolve(localesDir)}/${locale}.json`)
parsed = JSON.parse(content)
} catch (err) {}
} catch (err) {
if (err instanceof SyntaxError) {
// Check if there are merge conflicts in the JSON file
if (content.includes('<<<<<<<') && content.includes('>>>>>>>')) {
throw new SyntaxError(
`Error parsing ${locale}.json: merge conflicts found.\n\nPlease resolve merge conflicts before continuing.`,
)
}
throw err
}
}

parsed.__locale = locale

// Detect duplicate messages in the English locale and store them per message
// this will cause a non-zero exit code so that this issue can be caught in CI
if (locale === defaultLocale) {
const duplicates = Object.entries(parsed)
.reduce((acc, [id, message]) => {
const existing = acc.find(d => d.message === message)
if (existing) {
existing.ids.push(id)
} else {
acc.push({ message, ids: [id] })
}
return acc
}, [])
.filter(d => d.ids.length > 1)

if (duplicates.length > 0) {
// Write duplicates to a file
await write(
`${path.resolve(localesDir)}/${locale}-duplicates.json`,
JSON.stringify(duplicates, null, 2),
)

console.warn(
`Duplicate messages found in the "${locale}" locale. This means that the same message text is used more than once. See ${locale}-duplicates.json for details and resolve the issue before continuing.`,
)

if (!ignoreDuplicates) {
process.exit(1)
}
}
}

return parsed
}),
)

return loc.reduce(function (acc, next) {
return loc.reduce((acc, next) => {
const locale = next.__locale
delete next.__locale

Expand All @@ -181,18 +225,16 @@ const readLocales = async function () {
*
* @returns {object} - The messages, keyed by message id.
*/
const readMessages = async function () {
const readMessages = async () => {
if (!messagesDir) {
return {}
}
const files = await glob(`${path.resolve(messagesDir)}/**/*.json`)
return files
.map(f => fs.readFileSync(f, 'utf-8'))
.map(c => JSON.parse(c))
.reduce(function (acc, next) {
return [...acc, ...next]
}, [])
.reduce(function (acc, next) {
.reduce((acc, next) => [...acc, ...next], [])
.reduce((acc, next) => {
if (next.id in acc) {
console.warn(`message id ${next.id} seen multiple times`)
}
Expand All @@ -209,22 +251,23 @@ const readMessages = async function () {
*
* @returns {object} - The backend messages, keyed by message id.
*/
const readBackendMessages = async function () {
const readBackendMessages = async () => {
if (!backendMessages) {
return {}
}
const backend = JSON.parse(await read(`${path.resolve(backendMessages)}`))
return Object.keys(backend).reduce(function (acc, id) {
return {
return Object.keys(backend).reduce(
(acc, id) => ({
...acc,
[id]: {
id,
defaultMessage: backend[id].translations[defaultLocale],
locales: backend[id].translations,
description: backend[id].description,
},
}
}, {})
}),
{},
)
}

/**
Expand All @@ -234,7 +277,7 @@ const readBackendMessages = async function () {
* @param {Array<string>} pth - The path to find in the object.
* @returns {any} - The value of the key at the path, or null if not found.
*/
const get = function (object, ...pth) {
const get = (object, ...pth) => {
if (object === null) {
return null
}
Expand All @@ -256,17 +299,17 @@ const get = function (object, ...pth) {
* Write locales to their corresponding file in the localesDir (specified by --locales).
*
* @param {object} locales - The locales to write.
* @param backendIds
* @param {Array} backendIds - The ids of the backend messages.
* @returns {Promise<undefined>} - A promise that resolves when all locales have been written.
*/
const writeLocales = async function (locales, backendIds) {
return Promise.all(
Object.keys(locales).map(async function (key) {
const writeLocales = async (locales, backendIds) =>
Promise.all(
Object.keys(locales).map(async key => {
const locale = locales[key]
let cleaned = locale
if (key === defaultLocale && !backendOnly) {
// Remove backend keys from the default locale
cleaned = Object.keys(locale).reduce(function (acc, next) {
cleaned = Object.keys(locale).reduce((acc, next) => {
if (!backendIds.includes(next)) {
return {
...acc,
Expand All @@ -281,23 +324,22 @@ const writeLocales = async function (locales, backendIds) {
await write(`${localesDir}/${key}.json`, content)
}),
)
}

/**
* Write messages out for supported locales as CSV, for use when translating.
*
* @param {object} locales - The locales to write.
* @param targetLanguage - The language (locale) to translate to.
* @param {string} targetLanguage - The language (locale) to translate to.
* @returns {Promise<undefined>} - A promise that resolves when the file has been written.
*/
const writeInterpreterFile = async function (locales, targetLanguage) {
const writeInterpreterFile = async (locales, targetLanguage) => {
// Write the data as CSV, the first column being the message id
// the second column being the English message, and the third being the target language message.

const locale = locales[targetLanguage]
const content = Object.keys(locale)
.filter(id => !locale[id])
.map(function (id) {
.map(id => {
const message = locale[id] || ''
// Add proper escapes
const english = locales[defaultLocale][id].replace(/"/g, '""')
Expand All @@ -309,7 +351,7 @@ const writeInterpreterFile = async function (locales, targetLanguage) {
}

// Main function.
const main = async function () {
const main = async () => {
const [locales, messages, backend] = await Promise.all([
readLocales(),
readMessages(),
Expand Down Expand Up @@ -353,14 +395,14 @@ const main = async function () {

if (exportForTranslation) {
const supportWithoutDefault = support.filter(s => s !== defaultLocale)
supportWithoutDefault.forEach(async function (targetLanguage) {
supportWithoutDefault.forEach(async targetLanguage => {
await writeInterpreterFile(updated, targetLanguage)
})
}
console.log('Locale files updated.')
}

main().catch(function (err) {
main().catch(err => {
console.error(err)
process.exit(1)
})

0 comments on commit cc189e5

Please sign in to comment.