From 015c043a978cb5c2322d7a8866382eb057ad87bd Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Wed, 2 Oct 2019 16:07:01 -0400 Subject: [PATCH 1/7] Implement core of a tool that identifies unused i18n tokens --- tools/unused-i18n-token-finder.js | 130 ++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 tools/unused-i18n-token-finder.js diff --git a/tools/unused-i18n-token-finder.js b/tools/unused-i18n-token-finder.js new file mode 100644 index 000000000..11b3aca15 --- /dev/null +++ b/tools/unused-i18n-token-finder.js @@ -0,0 +1,130 @@ +/** + * i18n Unused Tokens Checker + * * Combs through default local messages.json file looking for tokens that are not used in the project + * * Writes the results to a file + * * Results should be manually verified before removing any tokens because some tokens are dynamically generated + * and this simple script does not account for that + * + * Ghostery Browser Extension + * http://www.ghostery.com/ + * + * Copyright 2019 Ghostery, Inc. All rights reserved. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0 + */ + +/* eslint import/no-extraneous-dependencies: 0 */ +/* eslint no-console: 0 */ + +const fs = require('fs-extra'); +const jsonfile = require('jsonfile') + +// Constants +const UNUSED_TOKENS_FILE = './tools/i18n_results/unused_tokens.txt'; + +// Empty tools/i18n_results directory +fs.emptyDirSync('./tools/i18n_results'); + + +/** + * Outputs the contents of an object to a .txt file. + * @param string fileName The location of the file we will output to + * @param object resultsObject An object with the data we will output + * @return none + */ + +function recordResults(fileName, resultsObject) { + const stream = fs.createWriteStream(fileName); + stream.once('open', () => { + Object.keys(resultsObject).forEach((key) => { + stream.write(`${key}[${resultsObject[key].length}]:\n`); + resultsObject[key].forEach((duplicate) => { + stream.write(`${duplicate}\n`); + }); + stream.write('\n'); + }); + stream.end(); + }); +} + +function findUnusedTokens(tokens, filepaths) { + tokens = tokens.map(token => ({ value: token, isUsed: false })); + + // Simpler than splicing arrays + filepaths.forEach((filepath) => { + const fileContents = fs.readFileSync(filepath, 'utf8'); + tokens.forEach((token) => { + if (token.isUsed) { return; } + + if (fileContents.includes(token.value)) { + token.isUsed = true; + } + }); + }); + + const unusedTokens = + (tokens.filter(token => token.isUsed === false)) + .map(token => token.value); + + return unusedTokens; +} + +/** + * Recursively collect the filepaths of files that + * satisfy the supplied extension and file system location conditions + * @param [Array|object] whereToLookAndForWhatExtensions + * @param [string Array] filepaths The matching filepaths + * @returns [string Array] filepaths The matching filepaths + */ +function getFilepaths(whereToLookAndForWhatExtensions, filepaths = []) { + const targets = whereToLookAndForWhatExtensions; + + if (Array.isArray(targets)) { + targets.forEach((target) => { + filepaths = getFilepaths(target, filepaths); + }); + } else { + const dirEntries = fs.readdirSync(targets.dir, { withFileTypes: true }); + + dirEntries.forEach((dirEntry) => { + if (dirEntry.isDirectory()) { + filepaths = getFilepaths({ + dir: `${targets.dir}/${dirEntry.name}`, + extensions: targets.extensions + }, filepaths); + } + if (dirEntry.isFile()) { + if (targets.extensions.some(extension => dirEntry.name.endsWith(extension))) { + filepaths.push(`${targets.dir}/${dirEntry.name}`); + }; + }; + }); + + return filepaths; + } + + return filepaths; +} + +function getJSONKeys(filepath) { + const json = jsonfile.readFileSync(filepath); + return Object.keys(json); +} + +console.time('i18n-unused-tokens-checker.js'); +// recordResults( + findUnusedTokens( + getJSONKeys('./_locales/en/messages.json'), + getFilepaths( + [ + // Overly broad, but we favor simplicity since there is no compelling reason here to favor performance / efficiency + // Also, we prefer that unused tokens be incorrectly reported as used (due to an overbroad search) than vice versa + { dir: './app', extensions: ['.jsx', '.js'] }, + { dir: './src', extensions: ['.js'] }, + ] + ) + ); +//); +console.timeEnd('i18n-unused-tokens-checker.js'); From 9064846375e7c7f26938bd4a0f220413afc9edc3 Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Thu, 3 Oct 2019 10:37:33 -0400 Subject: [PATCH 2/7] Clean up and document unused i18n token finder script --- tools/possibly-unused-i18n-token-finder.js | 118 +++++++++++++++++++ tools/unused-i18n-token-finder.js | 130 --------------------- 2 files changed, 118 insertions(+), 130 deletions(-) create mode 100644 tools/possibly-unused-i18n-token-finder.js delete mode 100644 tools/unused-i18n-token-finder.js diff --git a/tools/possibly-unused-i18n-token-finder.js b/tools/possibly-unused-i18n-token-finder.js new file mode 100644 index 000000000..bf01017fd --- /dev/null +++ b/tools/possibly-unused-i18n-token-finder.js @@ -0,0 +1,118 @@ +/** + * Possibly Unused i18n Token Finder + * Key word: POSSIBLY + * + * Looks for i18n tokens that MAY be unused by the code + * Since some tokens are generated dynamically, the list generated by this script + * should ALWAYS be verified manually before removing any of the tokens in it + * + * Ghostery Browser Extension + * http://www.ghostery.com/ + * + * Copyright 2019 Ghostery, Inc. All rights reserved. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0 + */ + +// eslint-disable-next-line import/no-extraneous-dependencies +const fs = require('fs'); +// eslint-disable-next-line import/no-extraneous-dependencies +const jsonfile = require('jsonfile'); + +// Constants +const DEFAULT_LOCALE_TOKENS_FILE = './_locales/en/messages.json'; +const UNUSED_TOKENS_FILE = './tools/i18n_results/unused_tokens.txt'; + +function saveListOfUnusedTokensToFile(filepath, unusedTokens) { + fs.writeFileSync( + filepath, + unusedTokens.join('\n') + ); +} + +function findUnusedTokens(tokens, filepaths) { + tokens = tokens.map(token => ({ value: token, isUsed: false })); + + filepaths.forEach((filepath) => { + const fileContents = fs.readFileSync(filepath, 'utf8'); + tokens.forEach((token) => { + if (token.isUsed) { return; } + + // THE TEST + if (fileContents.includes(`t('${token.value}`)) { + token.isUsed = true; + } + }); + }); + + const unusedTokens = + (tokens.filter(token => token.isUsed === false)) + .map(token => token.value); + + return unusedTokens; +} + +/** + * Recursively collect the filepaths of files that + * satisfy the supplied extension and file system location conditions + * @param [Array|object] whereToLookAndForWhatExtensions + * @param [string Array] filepaths The matching filepaths + * @returns [string Array] filepaths The matching filepaths + */ +function getFilepaths(whereToLookAndForWhatExtensions, filepaths = []) { + const target = whereToLookAndForWhatExtensions; + + if (Array.isArray(target)) { + target.forEach((t) => { + filepaths = getFilepaths(t, filepaths); + }); + } else { + const dirEntries = fs.readdirSync(target.dir, { withFileTypes: true }); + + dirEntries.forEach((dirEntry) => { + if (dirEntry.isDirectory()) { + filepaths = getFilepaths({ + dir: `${target.dir}/${dirEntry.name}`, + extensions: target.extensions + }, filepaths); + } else if (dirEntry.isFile()) { + if (target.extensions.some(extension => dirEntry.name.endsWith(extension))) { + filepaths.push(`${target.dir}/${dirEntry.name}`); + } + } + }); + + return filepaths; + } + + return filepaths; +} + +function getJSONKeys(filepath) { + const json = jsonfile.readFileSync(filepath); + return Object.keys(json); +} + +saveListOfUnusedTokensToFile( + UNUSED_TOKENS_FILE, + findUnusedTokens( + getJSONKeys(DEFAULT_LOCALE_TOKENS_FILE), + getFilepaths( + [ + // Overly broad, but we favor simplicity since there is no compelling reason here to favor performance / efficiency + // Also, we prefer that unused tokens be incorrectly reported as used than vice versa + { dir: './app', extensions: ['.jsx', '.js'] }, + { dir: './src', extensions: ['.js'] }, + ] + ) + ) +); + +console.log('\nPLEASE NOTE:'); +console.log('Since some i18n tokens are generated dynamically,') +console.log('and since some others are formatted in a non-standard way,'); +console.log('the list generated by this script should ALWAYS'); +console.log('be verified manually before removing any of the tokens in it.'); +console.log('\nThe results are in ./tools/i18n_results/unused_tokens.txt'); diff --git a/tools/unused-i18n-token-finder.js b/tools/unused-i18n-token-finder.js deleted file mode 100644 index 11b3aca15..000000000 --- a/tools/unused-i18n-token-finder.js +++ /dev/null @@ -1,130 +0,0 @@ -/** - * i18n Unused Tokens Checker - * * Combs through default local messages.json file looking for tokens that are not used in the project - * * Writes the results to a file - * * Results should be manually verified before removing any tokens because some tokens are dynamically generated - * and this simple script does not account for that - * - * Ghostery Browser Extension - * http://www.ghostery.com/ - * - * Copyright 2019 Ghostery, Inc. All rights reserved. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0 - */ - -/* eslint import/no-extraneous-dependencies: 0 */ -/* eslint no-console: 0 */ - -const fs = require('fs-extra'); -const jsonfile = require('jsonfile') - -// Constants -const UNUSED_TOKENS_FILE = './tools/i18n_results/unused_tokens.txt'; - -// Empty tools/i18n_results directory -fs.emptyDirSync('./tools/i18n_results'); - - -/** - * Outputs the contents of an object to a .txt file. - * @param string fileName The location of the file we will output to - * @param object resultsObject An object with the data we will output - * @return none - */ - -function recordResults(fileName, resultsObject) { - const stream = fs.createWriteStream(fileName); - stream.once('open', () => { - Object.keys(resultsObject).forEach((key) => { - stream.write(`${key}[${resultsObject[key].length}]:\n`); - resultsObject[key].forEach((duplicate) => { - stream.write(`${duplicate}\n`); - }); - stream.write('\n'); - }); - stream.end(); - }); -} - -function findUnusedTokens(tokens, filepaths) { - tokens = tokens.map(token => ({ value: token, isUsed: false })); - - // Simpler than splicing arrays - filepaths.forEach((filepath) => { - const fileContents = fs.readFileSync(filepath, 'utf8'); - tokens.forEach((token) => { - if (token.isUsed) { return; } - - if (fileContents.includes(token.value)) { - token.isUsed = true; - } - }); - }); - - const unusedTokens = - (tokens.filter(token => token.isUsed === false)) - .map(token => token.value); - - return unusedTokens; -} - -/** - * Recursively collect the filepaths of files that - * satisfy the supplied extension and file system location conditions - * @param [Array|object] whereToLookAndForWhatExtensions - * @param [string Array] filepaths The matching filepaths - * @returns [string Array] filepaths The matching filepaths - */ -function getFilepaths(whereToLookAndForWhatExtensions, filepaths = []) { - const targets = whereToLookAndForWhatExtensions; - - if (Array.isArray(targets)) { - targets.forEach((target) => { - filepaths = getFilepaths(target, filepaths); - }); - } else { - const dirEntries = fs.readdirSync(targets.dir, { withFileTypes: true }); - - dirEntries.forEach((dirEntry) => { - if (dirEntry.isDirectory()) { - filepaths = getFilepaths({ - dir: `${targets.dir}/${dirEntry.name}`, - extensions: targets.extensions - }, filepaths); - } - if (dirEntry.isFile()) { - if (targets.extensions.some(extension => dirEntry.name.endsWith(extension))) { - filepaths.push(`${targets.dir}/${dirEntry.name}`); - }; - }; - }); - - return filepaths; - } - - return filepaths; -} - -function getJSONKeys(filepath) { - const json = jsonfile.readFileSync(filepath); - return Object.keys(json); -} - -console.time('i18n-unused-tokens-checker.js'); -// recordResults( - findUnusedTokens( - getJSONKeys('./_locales/en/messages.json'), - getFilepaths( - [ - // Overly broad, but we favor simplicity since there is no compelling reason here to favor performance / efficiency - // Also, we prefer that unused tokens be incorrectly reported as used (due to an overbroad search) than vice versa - { dir: './app', extensions: ['.jsx', '.js'] }, - { dir: './src', extensions: ['.js'] }, - ] - ) - ); -//); -console.timeEnd('i18n-unused-tokens-checker.js'); From eb21b9f262dc300b41189fd73dece9428b21d715 Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Thu, 10 Oct 2019 16:32:33 -0400 Subject: [PATCH 3/7] Remove 'possibly' from unused i18n token finder script name --- ...ly-unused-i18n-token-finder.js => unused-i18n-token-finder.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{possibly-unused-i18n-token-finder.js => unused-i18n-token-finder.js} (100%) diff --git a/tools/possibly-unused-i18n-token-finder.js b/tools/unused-i18n-token-finder.js similarity index 100% rename from tools/possibly-unused-i18n-token-finder.js rename to tools/unused-i18n-token-finder.js From b20fd10ffc9422707c55513010369e6470bc6c13 Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Thu, 10 Oct 2019 16:45:46 -0400 Subject: [PATCH 4/7] Create i18n_results directory if needed when writing unused i18n token checker output --- tools/unused-i18n-token-finder.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/unused-i18n-token-finder.js b/tools/unused-i18n-token-finder.js index bf01017fd..8d05d8524 100644 --- a/tools/unused-i18n-token-finder.js +++ b/tools/unused-i18n-token-finder.js @@ -23,9 +23,16 @@ const jsonfile = require('jsonfile'); // Constants const DEFAULT_LOCALE_TOKENS_FILE = './_locales/en/messages.json'; -const UNUSED_TOKENS_FILE = './tools/i18n_results/unused_tokens.txt'; +const UNUSED_TOKENS_DIR = './tools/i18n_results'; +const UNUSED_TOKENS_FILENAME = 'unused_tokens.txt'; + +function saveListOfUnusedTokensToFile(unusedTokens) { + const filepath = `${UNUSED_TOKENS_DIR}/${UNUSED_TOKENS_FILENAME}`; + + if (!fs.existsSync(UNUSED_TOKENS_DIR)) { + fs.mkdirSync(UNUSED_TOKENS_DIR); + } -function saveListOfUnusedTokensToFile(filepath, unusedTokens) { fs.writeFileSync( filepath, unusedTokens.join('\n') @@ -96,7 +103,6 @@ function getJSONKeys(filepath) { } saveListOfUnusedTokensToFile( - UNUSED_TOKENS_FILE, findUnusedTokens( getJSONKeys(DEFAULT_LOCALE_TOKENS_FILE), getFilepaths( From 80e0adbb484ddf99ea8bce6da7c1966eff4a783c Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Thu, 10 Oct 2019 16:48:13 -0400 Subject: [PATCH 5/7] Remove a redundant return statement in unused i18n token finder --- tools/unused-i18n-token-finder.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/unused-i18n-token-finder.js b/tools/unused-i18n-token-finder.js index 8d05d8524..ba4ef860f 100644 --- a/tools/unused-i18n-token-finder.js +++ b/tools/unused-i18n-token-finder.js @@ -90,8 +90,6 @@ function getFilepaths(whereToLookAndForWhatExtensions, filepaths = []) { } } }); - - return filepaths; } return filepaths; From bab629a25e7504ff03011f2c6cdf1b7a91802b3e Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Thu, 10 Oct 2019 16:52:37 -0400 Subject: [PATCH 6/7] Add execution time logging to unused i18n token finder script --- tools/unused-i18n-token-finder.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/unused-i18n-token-finder.js b/tools/unused-i18n-token-finder.js index ba4ef860f..f78e79511 100644 --- a/tools/unused-i18n-token-finder.js +++ b/tools/unused-i18n-token-finder.js @@ -16,6 +16,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0 */ +console.time('unused-i18n-token-finder'); + // eslint-disable-next-line import/no-extraneous-dependencies const fs = require('fs'); // eslint-disable-next-line import/no-extraneous-dependencies @@ -119,4 +121,6 @@ console.log('Since some i18n tokens are generated dynamically,') console.log('and since some others are formatted in a non-standard way,'); console.log('the list generated by this script should ALWAYS'); console.log('be verified manually before removing any of the tokens in it.'); -console.log('\nThe results are in ./tools/i18n_results/unused_tokens.txt'); +console.log('\nThe results are in ./tools/i18n_results/unused_tokens.txt\n'); + +console.timeEnd('unused-i18n-token-finder'); From 1ff575f485f3f3b61a04c7d34c1dcb1a06415415 Mon Sep 17 00:00:00 2001 From: wlycdgr Date: Thu, 10 Oct 2019 17:07:36 -0400 Subject: [PATCH 7/7] Fix path bugs in the original i18n-checker script --- tools/i18n-checker.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/i18n-checker.js b/tools/i18n-checker.js index dbf91e0af..0f5d96f36 100644 --- a/tools/i18n-checker.js +++ b/tools/i18n-checker.js @@ -24,7 +24,7 @@ const oboe = require('oboe'); const LOCALES_FOLDER = './_locales'; const GATHER_FILE_PATHS_EXCEPTIONS = ['.DS_Store']; const LANG_FILES_COUNT = 14; -const DEFAULT_LOCALE_PATH = '../_locales/en/messages.json'; +const DEFAULT_LOCALE_PATH = './_locales/en/messages.json'; const DUPLICATE_TOKENS_FILE = './tools/i18n_results/duplicate_tokens.txt'; const MISSING_TOKENS_FILE = './tools/i18n_results/missing_tokens.txt'; const EXTRA_TOKENS_FILE = './tools/i18n_results/extra_tokens.txt'; @@ -175,7 +175,7 @@ function findMissingKeys(paths) { let hasMissingKeys = false; const missingKeys = {}; paths.forEach((path) => { - const localeJson = jsonfile.readFileSync(`.${path}`); + const localeJson = jsonfile.readFileSync(`${path}`); const locale = path.match(/_locales\/(.*)\/messages.json/)[1]; missingKeys[locale] = []; Object.keys(defaultLocaleJson).forEach((key) => { @@ -210,7 +210,7 @@ function findExtraKeys(paths) { let hasExtraKeys = false; const extraKeys = {}; paths.forEach((path) => { - const localeJson = jsonfile.readFileSync(`.${path}`); + const localeJson = jsonfile.readFileSync(`${path}`); const locale = path.match(/_locales\/(.*)\/messages.json/)[1]; extraKeys[locale] = []; Object.keys(localeJson).forEach((key) => { @@ -243,7 +243,7 @@ function findMalformedKeys(paths) { let hasMalformedKeys = false; const malformedKeys = {}; paths.forEach((path) => { - const localeJson = jsonfile.readFileSync(`.${path}`); + const localeJson = jsonfile.readFileSync(`${path}`); const locale = path.match(/_locales\/(.*)\/messages.json/)[1]; malformedKeys[locale] = []; Object.keys(localeJson).forEach((key) => { @@ -278,7 +278,7 @@ function findMissingPlaceholders(paths) { let hasMissingPlaceholders = false; const missingPlaceholders = {}; paths.forEach((path) => { - const localeJson = jsonfile.readFileSync(`.${path}`); + const localeJson = jsonfile.readFileSync(`${path}`); const locale = path.match(/_locales\/(.*)\/messages.json/)[1]; missingPlaceholders[locale] = []; Object.keys(defaultLocaleJson).forEach((key) => { @@ -322,7 +322,7 @@ function findExtraPlaceholders(paths) { let hasExtraPlaceholders = false; const extraPlaceholders = {}; paths.forEach((path) => { - const localeJson = jsonfile.readFileSync(`.${path}`); + const localeJson = jsonfile.readFileSync(`${path}`); const locale = path.match(/_locales\/(.*)\/messages.json/)[1]; extraPlaceholders[locale] = []; Object.keys(localeJson).forEach((key) => { @@ -364,7 +364,7 @@ function findMalformedPlaceholders(paths) { let hasMalformedPlaceholders = false; const malformedPlaceholders = []; paths.forEach((path) => { - const localeJson = jsonfile.readFileSync(`.${path}`); + const localeJson = jsonfile.readFileSync(`${path}`); const locale = path.match(/_locales\/(.*)\/messages.json/)[1]; malformedPlaceholders[locale] = []; Object.keys(localeJson).forEach((key) => {