From 4e859175b912608f786f28ad994f88319f07ea6d Mon Sep 17 00:00:00 2001 From: Caleb Richelson Date: Thu, 4 Jun 2020 12:17:38 +0200 Subject: [PATCH 1/3] Add tool to look for keys in app and src and make sure they are in /en/messages.json --- .gitignore | 1 + package.json | 1 + tools/i18n-checker.js | 8 ++- tools/token-checker.js | 144 +++++++++++++++++++++++++++++++++++++++++ yarn.lock | 25 +++++++ 5 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 tools/token-checker.js diff --git a/.gitignore b/.gitignore index c4e3ce895..72b027ea6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ yarn-error.log .ltk tools/i18n_results tools/leet/*.json +tools/token_results ## Cliqz cliqz/ diff --git a/package.json b/package.json index ee161a7d3..6474655c9 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "eslint-plugin-import": "^2.20.2", "eslint-plugin-jsx-a11y": "^6.2.3", "eslint-plugin-react": "^7.19.0", + "find-in-files": "^0.5.0", "fs-extra": "^9.0.0", "jest": "^25.4.0", "jest-fetch-mock": "^3.0.3", diff --git a/tools/i18n-checker.js b/tools/i18n-checker.js index 0f5d96f36..c0fc788ed 100644 --- a/tools/i18n-checker.js +++ b/tools/i18n-checker.js @@ -84,7 +84,13 @@ function validateJson(paths) { let hasError = false; paths.forEach((path) => { try { - jsonfile.readFileSync(`${path}`); + const file = jsonfile.readFileSync(`${path}`); + Object.keys(file).forEach((key) => { + if (!/^\w*$/.test(key)) { + hasError = true; + console.log('Error: file %s has invalid key "%s".', path, key); + } + }); } catch (err) { hasError = true; console.log('Error: file "%s" is not valid JSON.', path); diff --git a/tools/token-checker.js b/tools/token-checker.js new file mode 100644 index 000000000..0869a1914 --- /dev/null +++ b/tools/token-checker.js @@ -0,0 +1,144 @@ +/** + * Token Checker + * + * Ghostery Browser Extension + * http://www.ghostery.com/ + * + * Copyright 2020 Ghostery, Inc. All rights reserved. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0 + */ + +/* eslint import/no-extraneous-dependencies: 0 */ +/* eslint no-console: 0 */ + +console.time('token-checker'); + +const fs = require('fs-extra'); +const jsonfile = require('jsonfile'); +const findInFiles = require('find-in-files'); + +// Constants +const DEFAULT_LOCALE_PATH = './_locales/en/messages.json'; +const SEARCH_DIRECTORIES = ['app', 'databases', 'src', 'test', 'tools']; +const FIND_TOKEN_REGEX = /\Wt\(['|"|`](\w*)['|"|`]\)/; +const UNDEFINED_TOKEN_FILE = './tools/token_results/undefined_tokens.txt'; + +// Empty tools/token_results directory +fs.emptyDirSync('./tools/token_results'); + +/** + * Outputs the contents of an object to a .txt file. + * @param string fileName The location of the file we will output to + * @param object resultsObject An object with the data we will output + * @return none + */ +function recordResults(fileName, resultsObject) { + const stream = fs.createWriteStream(fileName); + stream.once('open', () => { + Object.keys(resultsObject).forEach((file) => { + stream.write(`'${file}' has missing tokens:\n`); + Object.keys(resultsObject[file]).forEach((token) => { + stream.write(`\t${token}\n`); + }); + stream.write('\n'); + }); + stream.end(); + }); +} + +/** + * Gathers the tokens used in a directory + * @param directory directory to search for tokens + * @const RegEx FIND_TOKEN_REGEX regex used to find tokens + * @return Promise Resolves with an object of all tokens and the files + * in which those tokens were found + */ +function findTokensInDirectory(directory) { + return new Promise((resolve) => { + const dirTokens = {}; + findInFiles.find(FIND_TOKEN_REGEX, directory).then((dirResults) => { + const fileNames = Object.keys(dirResults); + for (let i = 0; i < fileNames.length; i++) { + const fileName = fileNames[i]; + const fileMatches = dirResults[fileName].matches; + for (let j = 0; j < fileMatches.length; j++) { + const match = fileMatches[j]; + const token = match.substr(4, match.length - 6); + if (!dirTokens.hasOwnProperty(token)) { + dirTokens[token] = { files: {} }; + } + dirTokens[token].files[fileName] = true; + } + } + resolve(dirTokens); + }); + }); +} + +/** + * Gathers the tokens used in a directory + * @param dirsTokens Resolved object of findTokensInDirectory. + * Object of tokens and files in which the tokens were found. + * @const string DEFAULT_LOCALE_PATH The location of the default locale JSON file + * @return Promise Resolves with an object of files that have tokens not found + * in DEFAULT_LOCALE_PATH's messages.json file + */ +function compileUndefinedTokens(dirsTokens) { + const defaultLocaleJson = jsonfile.readFileSync(DEFAULT_LOCALE_PATH); + const undefinedTokenFiles = {}; + return new Promise((resolve) => { + for (let i = 0; i < dirsTokens.length; i++) { + const dirTokens = dirsTokens[i]; + const dirTokensArr = Object.keys(dirTokens); + for (let j = 0; j < dirTokensArr.length; j++) { + const token = dirTokensArr[j]; + if (!defaultLocaleJson.hasOwnProperty(token)) { + const files = Object.keys(dirTokens[token].files); + for (let k = 0; k < files.length; k++) { + const file = files[k]; + if (!undefinedTokenFiles.hasOwnProperty(file)) { + undefinedTokenFiles[file] = {}; + } + undefinedTokenFiles[file][token] = true; + } + } + } + } + resolve(undefinedTokenFiles); + }); +} + +/** + * Checks for missing tokens throughout the project. Writes the list of missing tokens to a file. + * Does not check for tokens that are defined using variables. + * @const array SEARCH_DIRECTORIES directories to search for tokens + * @const string UNDEFINED_TOKEN_FILE The file where we should write the tokens + * @return Promise Resolves or Rejects depending on whether there are undefined tokens + */ +function findUndefinedTokens() { + return new Promise((resolve, reject) => { + Promise.all( + SEARCH_DIRECTORIES.map(directory => findTokensInDirectory(directory)) + ).then(compileUndefinedTokens).then((undefinedTokenFiles) => { + const undefinedTokenFilesArr = Object.keys(undefinedTokenFiles); + if (undefinedTokenFilesArr.length >= 1) { + console.log('Error: undefined tokens were found. See them in `%s`.', UNDEFINED_TOKEN_FILE); + recordResults(UNDEFINED_TOKEN_FILE, undefinedTokenFiles); + reject(); + } else { + console.log('Scanned all directories for undefined tokens, none found.'); + resolve(); + } + }); + }); +} + +// Main +findUndefinedTokens().catch(() => { + console.log('Errors found. Fix the files and run `node tools/token-checker` to re-validate translation tokens.'); +}).then(() => { + console.timeEnd('token-checker'); +}); diff --git a/yarn.lock b/yarn.lock index e82b8ff67..90470417a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3569,6 +3569,14 @@ find-cache-dir@^2.1.0: make-dir "^2.0.0" pkg-dir "^3.0.0" +find-in-files@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/find-in-files/-/find-in-files-0.5.0.tgz#8e5a20ffb562e0a47cb916b7f7b821717025e691" + integrity sha512-VraTc6HdtdSHmAp0yJpAy20yPttGKzyBWc7b7FPnnsX9TOgmKx0g9xajizpF/iuu4IvNK4TP0SpyBT9zAlwG+g== + dependencies: + find "^0.1.5" + q "^1.0.1" + find-up@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" @@ -3599,6 +3607,13 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find@^0.1.5: + version "0.1.7" + resolved "https://registry.yarnpkg.com/find/-/find-0.1.7.tgz#c86c87af1ab18f222bbe38dec86cbc760d16a6fb" + integrity sha1-yGyHrxqxjyIrvjjeyGy8dg0Wpvs= + dependencies: + traverse-chain "~0.1.0" + findup-sync@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" @@ -6848,6 +6863,11 @@ punycode@^2.1.0, punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +q@^1.0.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" + integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= + qs@~6.5.2: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -8452,6 +8472,11 @@ tr46@^1.0.1: dependencies: punycode "^2.1.0" +traverse-chain@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/traverse-chain/-/traverse-chain-0.1.0.tgz#61dbc2d53b69ff6091a12a168fd7d433107e40f1" + integrity sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE= + treeify@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/treeify/-/treeify-1.1.0.tgz#4e31c6a463accd0943879f30667c4fdaff411bb8" From 8efcaaa4ff9599270dc42acf215525cbb357ff95 Mon Sep 17 00:00:00 2001 From: Caleb Richelson Date: Thu, 4 Jun 2020 12:22:15 +0200 Subject: [PATCH 2/3] Fix missing tokens --- _locales/en/messages.json | 3 +++ app/hub/Views/CreateAccountView/CreateAccountView.jsx | 2 +- app/panel/components/CreateAccount.jsx | 2 +- app/panel/components/StatsView.jsx | 2 +- app/panel/components/Subscribe.jsx | 2 +- app/panel/reducers/panel.js | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 1d7d86010..7a28e4a4a 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -704,6 +704,9 @@ "panel_detail_menu_premium_title": { "message": "Premium" }, + "panel_detail_learn_more": { + "message": "Learn More" + }, "panel_detail_history_title": { "message": "History" }, diff --git a/app/hub/Views/CreateAccountView/CreateAccountView.jsx b/app/hub/Views/CreateAccountView/CreateAccountView.jsx index af0667bde..7804b0c62 100644 --- a/app/hub/Views/CreateAccountView/CreateAccountView.jsx +++ b/app/hub/Views/CreateAccountView/CreateAccountView.jsx @@ -105,7 +105,7 @@ const CreateAccountView = (props) => { /> {confirmEmailError && (
- {t('your_emails_do_not_match')} + {t('your_email_do_not_match')}
)} diff --git a/app/panel/components/CreateAccount.jsx b/app/panel/components/CreateAccount.jsx index 7d0120504..17692ae2e 100644 --- a/app/panel/components/CreateAccount.jsx +++ b/app/panel/components/CreateAccount.jsx @@ -164,7 +164,7 @@ class CreateAccount extends React.Component { * -

{ t('your_emails_do_not_match') }

+

{ t('your_email_do_not_match') }

diff --git a/app/panel/components/StatsView.jsx b/app/panel/components/StatsView.jsx index 360ab1b68..93fc45917 100644 --- a/app/panel/components/StatsView.jsx +++ b/app/panel/components/StatsView.jsx @@ -196,7 +196,7 @@ const StatsView = (props) => {
-
{t('Get_Ghostery_Plus_bang')}
+
{t('get_ghostery_plus_bang')}
{ !loggedIn && (
diff --git a/app/panel/components/Subscribe.jsx b/app/panel/components/Subscribe.jsx index 81222e88a..40fe090b6 100644 --- a/app/panel/components/Subscribe.jsx +++ b/app/panel/components/Subscribe.jsx @@ -43,7 +43,7 @@ const Subscribe = (props) => { {t('subscribe_pitch_learn_more')}
- {t('Get_Ghostery_Plus_bang')} + {t('get_ghostery_plus_bang')}
{(loggedIn === 'false') && ( diff --git a/app/panel/reducers/panel.js b/app/panel/reducers/panel.js index 756db8c47..6b7fc0cf5 100644 --- a/app/panel/reducers/panel.js +++ b/app/panel/reducers/panel.js @@ -139,7 +139,7 @@ export default (state = initialState, action) => { errorText = t('email_address_in_use'); break; case '10080': - errorText = t('your_emails_do_not_match'); + errorText = t('your_email_do_not_match'); break; default: errorText = t('server_error_message'); From 6e2928cc63330c8f20c33e5574aeab7b9ccdc569 Mon Sep 17 00:00:00 2001 From: Caleb Richelson Date: Thu, 4 Jun 2020 12:31:28 +0200 Subject: [PATCH 3/3] Fix linting error and test error --- .../__tests__/__snapshots__/CreateAccountView.test.jsx.snap | 2 +- src/utils/click2play.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/hub/Views/CreateAccountView/__tests__/__snapshots__/CreateAccountView.test.jsx.snap b/app/hub/Views/CreateAccountView/__tests__/__snapshots__/CreateAccountView.test.jsx.snap index ca188b1ae..082755213 100644 --- a/app/hub/Views/CreateAccountView/__tests__/__snapshots__/CreateAccountView.test.jsx.snap +++ b/app/hub/Views/CreateAccountView/__tests__/__snapshots__/CreateAccountView.test.jsx.snap @@ -306,7 +306,7 @@ exports[`app/hub/Views/CreateAccount component Snapshot tests with react-test-re
- your_emails_do_not_match + your_email_do_not_match
diff --git a/src/utils/click2play.js b/src/utils/click2play.js index abd0ec7ce..037fc83dd 100644 --- a/src/utils/click2play.js +++ b/src/utils/click2play.js @@ -196,7 +196,6 @@ export function allowAllwaysC2P(app_id, tab_host) { // Remove fron site-specific-blocked if (conf.site_specific_blocks.hasOwnProperty(tab_host) && conf.site_specific_blocks[tab_host].includes(+app_id)) { - const index = conf.site_specific_blocks[tab_host].indexOf(+app_id); const { site_specific_blocks } = conf; site_specific_blocks[tab_host].splice(0, 1); conf.site_specific_blocks = site_specific_blocks;