From 34d96fa1ace0d81bbc2f3da26f2459f3f7884c3d Mon Sep 17 00:00:00 2001 From: Christopher Tino Date: Thu, 22 Aug 2019 18:23:06 -0400 Subject: [PATCH 1/6] build script for AMO reviewers --- tools/amo_builder.sh | 140 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 tools/amo_builder.sh diff --git a/tools/amo_builder.sh b/tools/amo_builder.sh new file mode 100755 index 000000000..37495a971 --- /dev/null +++ b/tools/amo_builder.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# +# Build script for AMO Reviewers +# +# Ghostery Browser Extension +# http://www.ghostery.com/ +# +# Copyright 2019 Ghostery, Inc. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0 + +set -e + +# Set directory paths for both projects +GHOSTERY_SOURCE_ZIP=`find . -name "ghostery-extension-*.zip"` +GHOSTERY_SOURCE_DIR="${GHOSTERY_SOURCE_ZIP%.zip*}" +CLIQZ_SOURCE_ZIP=`find . -name "browser-core-*.zip"` +CLIQZ_SOURCE_DIR="${CLIQZ_SOURCE_ZIP%.zip*}" + +# Extract +if [ ! -d $GHOSTERY_SOURCE_DIR ]; then + unzip $GHOSTERY_SOURCE_ZIP +fi + +if [ ! -d $CLIQZ_SOURCE_DIR ]; then + unzip $CLIQZ_SOURCE_ZIP +fi + +#### BROWSER CORE #### +cd $CLIQZ_SOURCE_DIR + +# Clean any previous builds +rm -fr build + +# Clean all the exiting node_modules for a more reproducible build +rm -fr node_modules + +# Install the exact versions from package-lock.json +npm ci + +# Build a production version of browser-core for Ghostery +./fern.js build configs/ghostery.js --no-maps --no-debug --environment=production + +echo "Browser Core build complete. Please see build/ directory." + +cd .. + +#### GHOSTERY EXTENSION #### +cd $GHOSTERY_SOURCE_DIR + +VERSION_FILE=manifest.json +MANIFEST=`cat $VERSION_FILE` +CWD=$(pwd) +VERSION=`cat $VERSION_FILE | jq '.version'` +BUILD_DIR=build +ZIP_FILE="$BUILD_DIR/ghostery-extension-v$VERSION.zip" + +# Install yarn +if ! type yarn > /dev/null; then + if ! type brew > /dev/null; then + brew install yarn + else + curl -o- -L https://yarnpkg.com/install.sh | bash + fi +fi + +# Install jq +if ! type jq > /dev/null; then + if ! type brew > /dev/null; then + brew install jq + else + abort "Please install jq: https://stedolan.github.io/jq/download/" + fi +fi + +# Clean any previous builds +rm -fr build + +# Clean all the exiting node_modules for a more reproducible build +rm -fr node_modules + +# Install local npm packages +yarn install --frozen-lockfile + +# Build for production +yarn build.prod + +# Clean up manifest.json +sed -i '' '/^\([[:space:]]*"debug": \).*$/d' $VERSION_FILE +sed -i '' '/^\([[:space:]]*"log": \).*$/d' $VERSION_FILE +sed -i '' '/^\([[:space:]]*"options_page": \).*$/d' $VERSION_FILE +sed -i '' '/^\([[:space:]]*"minimum_edge_version": \).*$/d' $VERSION_FILE +sed -i '' '/^\([[:space:]]*"minimum_chrome_version": \).*$/d' $VERSION_FILE +sed -i '' '/^\([[:space:]]*"minimum_opera_version": \).*$/d' $VERSION_FILE + +# Zip final build files +echo "Zipping to $BUILD_DIR/ directory" +test -d $BUILD_DIR || mkdir $BUILD_DIR && \ + zip --quiet -R "$ZIP_FILE" "*" -x \ + ^.*\ + app/content-scripts/\* \ + app/data-images/\* \ + app/panel/\* \ + app/setup/\* \ + app/scss/\* \ + app/licenses/\* \ + app/Account/\* \ + app/hub/\* \ + app/shared-components/\* \ + build/\* \ + dist/*.js.map \ + docs/\* \ + node_modules/\* \ + src/\* \ + test/\* \ + tools/\* \ + *.log$ \ + *.md$ \ + babel.config.js \ + CODEOWNERS \ + Dockerfile \ + Jenkinsfile \ + jest.config.js \ + jsdoc.json \ + package.json \ + package-lock.json \ + yarn.lock \ + webpack.* \ + *.DS_Store* \ + && \ + cd "$CWD" || \ + { + abort "Error occurred creating ghostery-extension zip" + } +echo "Zipped successfully into $BUILD_DIR/$ZIP_FILE" + +# Reset manifest +echo $MANIFEST > $VERSION_FILE From 445bb9c409c54fd778f479baa2a27956b6ad66fd Mon Sep 17 00:00:00 2001 From: Christopher Tino Date: Fri, 23 Aug 2019 17:04:10 -0400 Subject: [PATCH 2/6] AMO Builder: replace sed substitutions with jq for manifest.json. Clean up exlcudes --- tools/amo_builder.sh | 74 +++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/tools/amo_builder.sh b/tools/amo_builder.sh index 37495a971..f8dde129a 100755 --- a/tools/amo_builder.sh +++ b/tools/amo_builder.sh @@ -14,9 +14,9 @@ set -e # Set directory paths for both projects -GHOSTERY_SOURCE_ZIP=`find . -name "ghostery-extension-*.zip"` +GHOSTERY_SOURCE_ZIP=$(find . -name "ghostery-extension-*.zip") GHOSTERY_SOURCE_DIR="${GHOSTERY_SOURCE_ZIP%.zip*}" -CLIQZ_SOURCE_ZIP=`find . -name "browser-core-*.zip"` +CLIQZ_SOURCE_ZIP=$(find . -name "browser-core-*.zip") CLIQZ_SOURCE_DIR="${CLIQZ_SOURCE_ZIP%.zip*}" # Extract @@ -32,10 +32,10 @@ fi cd $CLIQZ_SOURCE_DIR # Clean any previous builds -rm -fr build +rm -rf build # Clean all the exiting node_modules for a more reproducible build -rm -fr node_modules +rm -rf node_modules # Install the exact versions from package-lock.json npm ci @@ -51,35 +51,37 @@ cd .. cd $GHOSTERY_SOURCE_DIR VERSION_FILE=manifest.json -MANIFEST=`cat $VERSION_FILE` -CWD=$(pwd) -VERSION=`cat $VERSION_FILE | jq '.version'` +MANIFEST_BACKUP=$(cat $VERSION_FILE) +RAW_VERSION=$(cat $VERSION_FILE | jq '.version') +VERSION=${RAW_VERSION//\"} # remove "" BUILD_DIR=build ZIP_FILE="$BUILD_DIR/ghostery-extension-v$VERSION.zip" +TMP_FILE=$(mktemp) -# Install yarn +# Check for yarn if ! type yarn > /dev/null; then - if ! type brew > /dev/null; then - brew install yarn - else - curl -o- -L https://yarnpkg.com/install.sh | bash - fi + abort "Please install yarn: https://yarnpkg.com/lang/en/docs/install/" fi -# Install jq +# Check for jq if ! type jq > /dev/null; then - if ! type brew > /dev/null; then - brew install jq - else - abort "Please install jq: https://stedolan.github.io/jq/download/" - fi + abort "Please install jq: https://stedolan.github.io/jq/download/" +fi + +# Check for nvm +. /usr/local/opt/nvm/nvm.sh +if ! command -v nvm | grep -q 'nvm'; then + abort "Please install nvm: https://github.com/nvm-sh/nvm" fi # Clean any previous builds -rm -fr build +rm -rf build # Clean all the exiting node_modules for a more reproducible build -rm -fr node_modules +rm -rf node_modules + +# Set node version +nvm use # Install local npm packages yarn install --frozen-lockfile @@ -87,22 +89,21 @@ yarn install --frozen-lockfile # Build for production yarn build.prod -# Clean up manifest.json -sed -i '' '/^\([[:space:]]*"debug": \).*$/d' $VERSION_FILE -sed -i '' '/^\([[:space:]]*"log": \).*$/d' $VERSION_FILE -sed -i '' '/^\([[:space:]]*"options_page": \).*$/d' $VERSION_FILE -sed -i '' '/^\([[:space:]]*"minimum_edge_version": \).*$/d' $VERSION_FILE -sed -i '' '/^\([[:space:]]*"minimum_chrome_version": \).*$/d' $VERSION_FILE -sed -i '' '/^\([[:space:]]*"minimum_opera_version": \).*$/d' $VERSION_FILE +# Clean up properties from manifest.json +cat $VERSION_FILE | jq 'del(.version_name, .debug, .log, .options_page, .minimum_edge_version, .minimum_chrome_version, .minimum_opera_version, .permissions[7,8], .background.persistent)' > ${TMP_FILE} +cat ${TMP_FILE} > $VERSION_FILE # copy into manifest.json +rm -f ${TMP_FILE} # Zip final build files -echo "Zipping to $BUILD_DIR/ directory" +echo "Zipping to $(pwd)/$BUILD_DIR/" test -d $BUILD_DIR || mkdir $BUILD_DIR && \ zip --quiet -R "$ZIP_FILE" "*" -x \ - ^.*\ + .* \ + .*/\* \ app/content-scripts/\* \ app/data-images/\* \ app/panel/\* \ + app/panel-android/\* \ app/setup/\* \ app/scss/\* \ app/licenses/\* \ @@ -116,8 +117,8 @@ test -d $BUILD_DIR || mkdir $BUILD_DIR && \ src/\* \ test/\* \ tools/\* \ - *.log$ \ - *.md$ \ + *.log \ + *.md \ babel.config.js \ CODEOWNERS \ Dockerfile \ @@ -128,13 +129,8 @@ test -d $BUILD_DIR || mkdir $BUILD_DIR && \ package-lock.json \ yarn.lock \ webpack.* \ - *.DS_Store* \ - && \ - cd "$CWD" || \ - { - abort "Error occurred creating ghostery-extension zip" - } + *.DS_Store* echo "Zipped successfully into $BUILD_DIR/$ZIP_FILE" # Reset manifest -echo $MANIFEST > $VERSION_FILE +echo $MANIFEST_BACKUP > $VERSION_FILE From 7806e5debbbc613ec63970a88525a95d35ba4cba Mon Sep 17 00:00:00 2001 From: Christopher Tino Date: Fri, 23 Aug 2019 23:07:40 -0400 Subject: [PATCH 3/6] create source code generator script --- .gitignore | 3 ++ tools/amo/README.md | 42 ++++++++++++++++++++++++++ tools/{amo_builder.sh => amo/build.sh} | 4 +-- tools/amo/generate.sh | 33 ++++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 tools/amo/README.md rename tools/{amo_builder.sh => amo/build.sh} (97%) create mode 100755 tools/amo/generate.sh diff --git a/.gitignore b/.gitignore index 2a9bbcb68..3e9c8d5ff 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ tools/leet/*.json # Cliqz cliqz/ +# Tools +tools/amo/*.zip + ## JSDoc docs/ jsdocs/ diff --git a/tools/amo/README.md b/tools/amo/README.md new file mode 100644 index 000000000..b44ff4759 --- /dev/null +++ b/tools/amo/README.md @@ -0,0 +1,42 @@ +# Ghostery Source Code Builder for AMO Review + +This package includes source code for the Ghostery extension along with Cliqz browser-core. Although browser-core is included in the Ghostery project as an NPM dependency, we provide the full source code here for easier review. + +## `build.sh` + +Use this script to install dependencies and build production code for the browser-core and ghostery-extension projects. + +#### Requirements + +This build script was tested on macOS Mojave with the following configuration: + ++ [Homebrew](https://brew.sh/) ++ [nvm](https://github.com/nvm-sh/nvm") + + `brew install nvm` ++ [Yarn](https://yarnpkg.com/) + + `brew install yarn` ++ [jq](https://stedolan.github.io/jq/) + + `brew install jq` + +#### Notes on `browser-core-X.X.X` + +After running `build.sh`, you should see the production files in the `browser-core-X.X.X/build/` directory. This should match the code found in `ghostery-extension-X.X.X/node_modules/browser-core/build/`. + +#### Notes on `ghostery-extension-X.X.X` + +After running `build.sh`, you should see the production build archive in the `ghostery-extension-X.X.X/build/` directory. This file should match the archive that was submitted to AMO. + +Compiled assets live in the `ghostery-extension-X.X.X/dist/` folder. Assets from Cliqz browser-core (see above) are copied into the `ghostery-extension-X.X.X/cliqz/` directory from `node_modules/browser-core/build/assets` using the `vendor-copy` npm package. That copy is triggered when running `yarn install`. See `package.json`: + +```json +"vendorCopy": [ + { + "from": "node_modules/browser-core/build/assets", + "to": "cliqz" + } +] +``` + +## `generate.sh` (Not included in submission) + +Fetches source code from GitHub and packages it along with `build.sh` and this README. This creates the complete source code archive that Ghostery sends along with its AMO submissions. diff --git a/tools/amo_builder.sh b/tools/amo/build.sh similarity index 97% rename from tools/amo_builder.sh rename to tools/amo/build.sh index f8dde129a..bc05b3b1c 100755 --- a/tools/amo_builder.sh +++ b/tools/amo/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Build script for AMO Reviewers +# Build script for AMO reviewers # # Ghostery Browser Extension # http://www.ghostery.com/ @@ -69,7 +69,7 @@ if ! type jq > /dev/null; then fi # Check for nvm -. /usr/local/opt/nvm/nvm.sh +source /usr/local/opt/nvm/nvm.sh if ! command -v nvm | grep -q 'nvm'; then abort "Please install nvm: https://github.com/nvm-sh/nvm" fi diff --git a/tools/amo/generate.sh b/tools/amo/generate.sh new file mode 100755 index 000000000..3fb76efe2 --- /dev/null +++ b/tools/amo/generate.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Generate source code archive for AMO reviewers +# +# Ghostery Browser Extension +# http://www.ghostery.com/ +# +# Copyright 2019 Ghostery, Inc. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0 + +VERSION_FILE=../../manifest.json +PACKAGE_FILE=../../package.json +ZIP_FILE=ghostery-source + +# Get version numbers +GHOSTERY_RAW_VERSION=$(cat $VERSION_FILE | jq '.version') +GHOSTERY_VERSION=${GHOSTERY_RAW_VERSION//\"} # remove "" +CLIQZ_BROWSER_CORE=$(cat $PACKAGE_FILE | jq '.dependencies["browser-core"]') +CLIQZ_VERSION=$(echo $CLIQZ_BROWSER_CORE | perl -pe '($_)=/([0-9]+([.][0-9]+)+)/') + +# Download source code zip files from GitHub +curl "https://github.com/ghostery/ghostery-extension/archive/v$GHOSTERY_VERSION.zip" -O -J -L --fail +curl "https://github.com/cliqz-oss/browser-core/archive/v$CLIQZ_VERSION.zip" -O -J -L --fail + +# Make source-code zip +zip --quiet -R "$ZIP_FILE" "*" -x generate.sh *.DS_Store + +# Clean up +rm "browser-core-$CLIQZ_VERSION.zip" +rm "ghostery-extension-$GHOSTERY_VERSION.zip" From eb4500d0fc0fccd6fa9bc7c6f45fd4f430089415 Mon Sep 17 00:00:00 2001 From: Christopher Tino Date: Sat, 24 Aug 2019 12:25:42 -0400 Subject: [PATCH 4/6] update AMO readme and remove sourcemaps from builder --- README.md | 7 +++---- tools/amo/README.md | 2 +- tools/amo/build.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b3b969270..f9fad0a23 100644 --- a/README.md +++ b/README.md @@ -135,12 +135,11 @@ Ghostery implements the following open-source products from [Cliqz](https://cliq + [GitHub](https://github.com/cliqz-oss/browser-core/blob/master/modules/offers-v2) ### Building Cliqz Modules for Ghostery -Cliqz modules are pre-built and included under the `browser-core` NPM dependency in [package.json](package.json). To reproduce this build process, grab the appropriate Ghostery release (v7.x.x) from the [browser-core](https://github.com/cliqz-oss/browser-core/releases) project. +Cliqz modules are pre-built and included under the `browser-core` NPM dependency in [package.json](package.json). To reproduce this build process, download the appropriate Ghostery release (v7.x.x) from the [browser-core](https://github.com/cliqz-oss/browser-core/releases) project. ```sh -$ npm install -$ ./fern.js build configs/ghostery.js --no-maps --environment=production -$ ./fern.js pack configs/ghostery.js +$ npm ci +$ ./fern.js build configs/ghostery.js --no-maps --no-debug --environment=production ``` ## Compatibility diff --git a/tools/amo/README.md b/tools/amo/README.md index b44ff4759..dd129c739 100644 --- a/tools/amo/README.md +++ b/tools/amo/README.md @@ -37,6 +37,6 @@ Compiled assets live in the `ghostery-extension-X.X.X/dist/` folder. Assets from ] ``` -## `generate.sh` (Not included in submission) +## `generate.sh` Fetches source code from GitHub and packages it along with `build.sh` and this README. This creates the complete source code archive that Ghostery sends along with its AMO submissions. diff --git a/tools/amo/build.sh b/tools/amo/build.sh index bc05b3b1c..026a2f5d6 100755 --- a/tools/amo/build.sh +++ b/tools/amo/build.sh @@ -111,13 +111,13 @@ test -d $BUILD_DIR || mkdir $BUILD_DIR && \ app/hub/\* \ app/shared-components/\* \ build/\* \ - dist/*.js.map \ docs/\* \ node_modules/\* \ src/\* \ test/\* \ tools/\* \ *.log \ + *.map \ *.md \ babel.config.js \ CODEOWNERS \ From 2c9dfcb60045a180c6ff3b5aedf4cdd4cf875726 Mon Sep 17 00:00:00 2001 From: Christopher Tino Date: Sat, 24 Aug 2019 12:41:59 -0400 Subject: [PATCH 5/6] update AMO readme --- tools/amo/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/amo/README.md b/tools/amo/README.md index dd129c739..6d5a81180 100644 --- a/tools/amo/README.md +++ b/tools/amo/README.md @@ -4,7 +4,7 @@ This package includes source code for the Ghostery extension along with Cliqz br ## `build.sh` -Use this script to install dependencies and build production code for the browser-core and ghostery-extension projects. +Use this script to install dependencies and build production code for the browser-core and ghostery-extension projects. Intended to be run inside the `ghostery-source` directory generated by `generate.sh`. #### Requirements From 09289f9409725b5a24756b11ef37b94777e38220 Mon Sep 17 00:00:00 2001 From: Christopher Tino Date: Mon, 26 Aug 2019 14:59:09 -0400 Subject: [PATCH 6/6] add nvm install version for ghostery --- README.md | 2 +- tools/amo/build.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f9fad0a23..0cb0ee8ab 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Cliqz modules are pre-built and included under the `browser-core` NPM dependency ```sh $ npm ci -$ ./fern.js build configs/ghostery.js --no-maps --no-debug --environment=production +$ ./fern.js build configs/ghostery.js --environment=production --no-debug ``` ## Compatibility diff --git a/tools/amo/build.sh b/tools/amo/build.sh index 026a2f5d6..4b3b3012d 100755 --- a/tools/amo/build.sh +++ b/tools/amo/build.sh @@ -41,7 +41,7 @@ rm -rf node_modules npm ci # Build a production version of browser-core for Ghostery -./fern.js build configs/ghostery.js --no-maps --no-debug --environment=production +./fern.js build configs/ghostery.js --environment=production --no-debug echo "Browser Core build complete. Please see build/ directory." @@ -81,6 +81,7 @@ rm -rf build rm -rf node_modules # Set node version +nvm install lts/carbon nvm use # Install local npm packages