diff --git a/.gitignore b/.gitignore index c4e3ce895..72b027ea6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ yarn-error.log .ltk tools/i18n_results tools/leet/*.json +tools/token_results ## Cliqz cliqz/ diff --git a/package.json b/package.json index ade3fd5ce..f2d09f4db 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "eslint-plugin-import": "^2.20.2", "eslint-plugin-jsx-a11y": "^6.2.3", "eslint-plugin-react": "^7.19.0", + "find-in-files": "^0.5.0", "fs-extra": "^9.0.0", "jest": "^25.4.0", "jest-fetch-mock": "^3.0.3", diff --git a/tools/i18n-checker.js b/tools/i18n-checker.js index 0f5d96f36..c0fc788ed 100644 --- a/tools/i18n-checker.js +++ b/tools/i18n-checker.js @@ -84,7 +84,13 @@ function validateJson(paths) { let hasError = false; paths.forEach((path) => { try { - jsonfile.readFileSync(`${path}`); + const file = jsonfile.readFileSync(`${path}`); + Object.keys(file).forEach((key) => { + if (!/^\w*$/.test(key)) { + hasError = true; + console.log('Error: file %s has invalid key "%s".', path, key); + } + }); } catch (err) { hasError = true; console.log('Error: file "%s" is not valid JSON.', path); diff --git a/tools/token-checker.js b/tools/token-checker.js new file mode 100644 index 000000000..0869a1914 --- /dev/null +++ b/tools/token-checker.js @@ -0,0 +1,144 @@ +/** + * Token Checker + * + * Ghostery Browser Extension + * http://www.ghostery.com/ + * + * Copyright 2020 Ghostery, Inc. All rights reserved. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0 + */ + +/* eslint import/no-extraneous-dependencies: 0 */ +/* eslint no-console: 0 */ + +console.time('token-checker'); + +const fs = require('fs-extra'); +const jsonfile = require('jsonfile'); +const findInFiles = require('find-in-files'); + +// Constants +const DEFAULT_LOCALE_PATH = './_locales/en/messages.json'; +const SEARCH_DIRECTORIES = ['app', 'databases', 'src', 'test', 'tools']; +const FIND_TOKEN_REGEX = /\Wt\(['|"|`](\w*)['|"|`]\)/; +const UNDEFINED_TOKEN_FILE = './tools/token_results/undefined_tokens.txt'; + +// Empty tools/token_results directory +fs.emptyDirSync('./tools/token_results'); + +/** + * Outputs the contents of an object to a .txt file. + * @param string fileName The location of the file we will output to + * @param object resultsObject An object with the data we will output + * @return none + */ +function recordResults(fileName, resultsObject) { + const stream = fs.createWriteStream(fileName); + stream.once('open', () => { + Object.keys(resultsObject).forEach((file) => { + stream.write(`'${file}' has missing tokens:\n`); + Object.keys(resultsObject[file]).forEach((token) => { + stream.write(`\t${token}\n`); + }); + stream.write('\n'); + }); + stream.end(); + }); +} + +/** + * Gathers the tokens used in a directory + * @param directory directory to search for tokens + * @const RegEx FIND_TOKEN_REGEX regex used to find tokens + * @return Promise Resolves with an object of all tokens and the files + * in which those tokens were found + */ +function findTokensInDirectory(directory) { + return new Promise((resolve) => { + const dirTokens = {}; + findInFiles.find(FIND_TOKEN_REGEX, directory).then((dirResults) => { + const fileNames = Object.keys(dirResults); + for (let i = 0; i < fileNames.length; i++) { + const fileName = fileNames[i]; + const fileMatches = dirResults[fileName].matches; + for (let j = 0; j < fileMatches.length; j++) { + const match = fileMatches[j]; + const token = match.substr(4, match.length - 6); + if (!dirTokens.hasOwnProperty(token)) { + dirTokens[token] = { files: {} }; + } + dirTokens[token].files[fileName] = true; + } + } + resolve(dirTokens); + }); + }); +} + +/** + * Gathers the tokens used in a directory + * @param dirsTokens Resolved object of findTokensInDirectory. + * Object of tokens and files in which the tokens were found. + * @const string DEFAULT_LOCALE_PATH The location of the default locale JSON file + * @return Promise Resolves with an object of files that have tokens not found + * in DEFAULT_LOCALE_PATH's messages.json file + */ +function compileUndefinedTokens(dirsTokens) { + const defaultLocaleJson = jsonfile.readFileSync(DEFAULT_LOCALE_PATH); + const undefinedTokenFiles = {}; + return new Promise((resolve) => { + for (let i = 0; i < dirsTokens.length; i++) { + const dirTokens = dirsTokens[i]; + const dirTokensArr = Object.keys(dirTokens); + for (let j = 0; j < dirTokensArr.length; j++) { + const token = dirTokensArr[j]; + if (!defaultLocaleJson.hasOwnProperty(token)) { + const files = Object.keys(dirTokens[token].files); + for (let k = 0; k < files.length; k++) { + const file = files[k]; + if (!undefinedTokenFiles.hasOwnProperty(file)) { + undefinedTokenFiles[file] = {}; + } + undefinedTokenFiles[file][token] = true; + } + } + } + } + resolve(undefinedTokenFiles); + }); +} + +/** + * Checks for missing tokens throughout the project. Writes the list of missing tokens to a file. + * Does not check for tokens that are defined using variables. + * @const array SEARCH_DIRECTORIES directories to search for tokens + * @const string UNDEFINED_TOKEN_FILE The file where we should write the tokens + * @return Promise Resolves or Rejects depending on whether there are undefined tokens + */ +function findUndefinedTokens() { + return new Promise((resolve, reject) => { + Promise.all( + SEARCH_DIRECTORIES.map(directory => findTokensInDirectory(directory)) + ).then(compileUndefinedTokens).then((undefinedTokenFiles) => { + const undefinedTokenFilesArr = Object.keys(undefinedTokenFiles); + if (undefinedTokenFilesArr.length >= 1) { + console.log('Error: undefined tokens were found. See them in `%s`.', UNDEFINED_TOKEN_FILE); + recordResults(UNDEFINED_TOKEN_FILE, undefinedTokenFiles); + reject(); + } else { + console.log('Scanned all directories for undefined tokens, none found.'); + resolve(); + } + }); + }); +} + +// Main +findUndefinedTokens().catch(() => { + console.log('Errors found. Fix the files and run `node tools/token-checker` to re-validate translation tokens.'); +}).then(() => { + console.timeEnd('token-checker'); +}); diff --git a/yarn.lock b/yarn.lock index e82b8ff67..90470417a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3569,6 +3569,14 @@ find-cache-dir@^2.1.0: make-dir "^2.0.0" pkg-dir "^3.0.0" +find-in-files@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/find-in-files/-/find-in-files-0.5.0.tgz#8e5a20ffb562e0a47cb916b7f7b821717025e691" + integrity sha512-VraTc6HdtdSHmAp0yJpAy20yPttGKzyBWc7b7FPnnsX9TOgmKx0g9xajizpF/iuu4IvNK4TP0SpyBT9zAlwG+g== + dependencies: + find "^0.1.5" + q "^1.0.1" + find-up@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" @@ -3599,6 +3607,13 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find@^0.1.5: + version "0.1.7" + resolved "https://registry.yarnpkg.com/find/-/find-0.1.7.tgz#c86c87af1ab18f222bbe38dec86cbc760d16a6fb" + integrity sha1-yGyHrxqxjyIrvjjeyGy8dg0Wpvs= + dependencies: + traverse-chain "~0.1.0" + findup-sync@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" @@ -6848,6 +6863,11 @@ punycode@^2.1.0, punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +q@^1.0.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" + integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= + qs@~6.5.2: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -8452,6 +8472,11 @@ tr46@^1.0.1: dependencies: punycode "^2.1.0" +traverse-chain@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/traverse-chain/-/traverse-chain-0.1.0.tgz#61dbc2d53b69ff6091a12a168fd7d433107e40f1" + integrity sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE= + treeify@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/treeify/-/treeify-1.1.0.tgz#4e31c6a463accd0943879f30667c4fdaff411bb8"