diff --git a/.gitignore b/.gitignore index 2a9bbcb68..3e9c8d5ff 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ tools/leet/*.json # Cliqz cliqz/ +# Tools +tools/amo/*.zip + ## JSDoc docs/ jsdocs/ diff --git a/README.md b/README.md index 51ba0e5ec..944ccef7a 100644 --- a/README.md +++ b/README.md @@ -133,12 +133,11 @@ Ghostery implements the following open-source products from [Cliqz](https://cliq + [GitHub](https://github.com/cliqz-oss/browser-core/blob/master/modules/offers-v2) ### Building Cliqz Modules for Ghostery -Cliqz modules are pre-built and included under the `browser-core` NPM dependency in [package.json](package.json). To reproduce this build process, grab the appropriate Ghostery release (v7.x.x) from the [browser-core](https://github.com/cliqz-oss/browser-core/releases) project. +Cliqz modules are pre-built and included under the `browser-core` NPM dependency in [package.json](package.json). To reproduce this build process, download the appropriate Ghostery release (v7.x.x) from the [browser-core](https://github.com/cliqz-oss/browser-core/releases) project. ```sh -$ npm install -$ ./fern.js build configs/ghostery.js --no-maps --environment=production -$ ./fern.js pack configs/ghostery.js +$ npm ci +$ ./fern.js build configs/ghostery.js --environment=production --no-debug ``` ## Compatibility diff --git a/tools/amo/README.md b/tools/amo/README.md new file mode 100644 index 000000000..6d5a81180 --- /dev/null +++ b/tools/amo/README.md @@ -0,0 +1,42 @@ +# Ghostery Source Code Builder for AMO Review + +This package includes source code for the Ghostery extension along with Cliqz browser-core. Although browser-core is included in the Ghostery project as an NPM dependency, we provide the full source code here for easier review. + +## `build.sh` + +Use this script to install dependencies and build production code for the browser-core and ghostery-extension projects. Intended to be run inside the `ghostery-source` directory generated by `generate.sh`. + +#### Requirements + +This build script was tested on macOS Mojave with the following configuration: + ++ [Homebrew](https://brew.sh/) ++ [nvm](https://github.com/nvm-sh/nvm") + + `brew install nvm` ++ [Yarn](https://yarnpkg.com/) + + `brew install yarn` ++ [jq](https://stedolan.github.io/jq/) + + `brew install jq` + +#### Notes on `browser-core-X.X.X` + +After running `build.sh`, you should see the production files in the `browser-core-X.X.X/build/` directory. This should match the code found in `ghostery-extension-X.X.X/node_modules/browser-core/build/`. + +#### Notes on `ghostery-extension-X.X.X` + +After running `build.sh`, you should see the production build archive in the `ghostery-extension-X.X.X/build/` directory. This file should match the archive that was submitted to AMO. + +Compiled assets live in the `ghostery-extension-X.X.X/dist/` folder. Assets from Cliqz browser-core (see above) are copied into the `ghostery-extension-X.X.X/cliqz/` directory from `node_modules/browser-core/build/assets` using the `vendor-copy` npm package. That copy is triggered when running `yarn install`. See `package.json`: + +```json +"vendorCopy": [ + { + "from": "node_modules/browser-core/build/assets", + "to": "cliqz" + } +] +``` + +## `generate.sh` + +Fetches source code from GitHub and packages it along with `build.sh` and this README. This creates the complete source code archive that Ghostery sends along with its AMO submissions. diff --git a/tools/amo/build.sh b/tools/amo/build.sh new file mode 100755 index 000000000..4b3b3012d --- /dev/null +++ b/tools/amo/build.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Build script for AMO reviewers +# +# Ghostery Browser Extension +# http://www.ghostery.com/ +# +# Copyright 2019 Ghostery, Inc. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0 + +set -e + +# Set directory paths for both projects +GHOSTERY_SOURCE_ZIP=$(find . -name "ghostery-extension-*.zip") +GHOSTERY_SOURCE_DIR="${GHOSTERY_SOURCE_ZIP%.zip*}" +CLIQZ_SOURCE_ZIP=$(find . -name "browser-core-*.zip") +CLIQZ_SOURCE_DIR="${CLIQZ_SOURCE_ZIP%.zip*}" + +# Extract +if [ ! -d $GHOSTERY_SOURCE_DIR ]; then + unzip $GHOSTERY_SOURCE_ZIP +fi + +if [ ! -d $CLIQZ_SOURCE_DIR ]; then + unzip $CLIQZ_SOURCE_ZIP +fi + +#### BROWSER CORE #### +cd $CLIQZ_SOURCE_DIR + +# Clean any previous builds +rm -rf build + +# Clean all the exiting node_modules for a more reproducible build +rm -rf node_modules + +# Install the exact versions from package-lock.json +npm ci + +# Build a production version of browser-core for Ghostery +./fern.js build configs/ghostery.js --environment=production --no-debug + +echo "Browser Core build complete. Please see build/ directory." + +cd .. + +#### GHOSTERY EXTENSION #### +cd $GHOSTERY_SOURCE_DIR + +VERSION_FILE=manifest.json +MANIFEST_BACKUP=$(cat $VERSION_FILE) +RAW_VERSION=$(cat $VERSION_FILE | jq '.version') +VERSION=${RAW_VERSION//\"} # remove "" +BUILD_DIR=build +ZIP_FILE="$BUILD_DIR/ghostery-extension-v$VERSION.zip" +TMP_FILE=$(mktemp) + +# Check for yarn +if ! type yarn > /dev/null; then + abort "Please install yarn: https://yarnpkg.com/lang/en/docs/install/" +fi + +# Check for jq +if ! type jq > /dev/null; then + abort "Please install jq: https://stedolan.github.io/jq/download/" +fi + +# Check for nvm +source /usr/local/opt/nvm/nvm.sh +if ! command -v nvm | grep -q 'nvm'; then + abort "Please install nvm: https://github.com/nvm-sh/nvm" +fi + +# Clean any previous builds +rm -rf build + +# Clean all the exiting node_modules for a more reproducible build +rm -rf node_modules + +# Set node version +nvm install lts/carbon +nvm use + +# Install local npm packages +yarn install --frozen-lockfile + +# Build for production +yarn build.prod + +# Clean up properties from manifest.json +cat $VERSION_FILE | jq 'del(.version_name, .debug, .log, .options_page, .minimum_edge_version, .minimum_chrome_version, .minimum_opera_version, .permissions[7,8], .background.persistent)' > ${TMP_FILE} +cat ${TMP_FILE} > $VERSION_FILE # copy into manifest.json +rm -f ${TMP_FILE} + +# Zip final build files +echo "Zipping to $(pwd)/$BUILD_DIR/" +test -d $BUILD_DIR || mkdir $BUILD_DIR && \ + zip --quiet -R "$ZIP_FILE" "*" -x \ + .* \ + .*/\* \ + app/content-scripts/\* \ + app/data-images/\* \ + app/panel/\* \ + app/panel-android/\* \ + app/setup/\* \ + app/scss/\* \ + app/licenses/\* \ + app/Account/\* \ + app/hub/\* \ + app/shared-components/\* \ + build/\* \ + docs/\* \ + node_modules/\* \ + src/\* \ + test/\* \ + tools/\* \ + *.log \ + *.map \ + *.md \ + babel.config.js \ + CODEOWNERS \ + Dockerfile \ + Jenkinsfile \ + jest.config.js \ + jsdoc.json \ + package.json \ + package-lock.json \ + yarn.lock \ + webpack.* \ + *.DS_Store* +echo "Zipped successfully into $BUILD_DIR/$ZIP_FILE" + +# Reset manifest +echo $MANIFEST_BACKUP > $VERSION_FILE diff --git a/tools/amo/generate.sh b/tools/amo/generate.sh new file mode 100755 index 000000000..3fb76efe2 --- /dev/null +++ b/tools/amo/generate.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Generate source code archive for AMO reviewers +# +# Ghostery Browser Extension +# http://www.ghostery.com/ +# +# Copyright 2019 Ghostery, Inc. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0 + +VERSION_FILE=../../manifest.json +PACKAGE_FILE=../../package.json +ZIP_FILE=ghostery-source + +# Get version numbers +GHOSTERY_RAW_VERSION=$(cat $VERSION_FILE | jq '.version') +GHOSTERY_VERSION=${GHOSTERY_RAW_VERSION//\"} # remove "" +CLIQZ_BROWSER_CORE=$(cat $PACKAGE_FILE | jq '.dependencies["browser-core"]') +CLIQZ_VERSION=$(echo $CLIQZ_BROWSER_CORE | perl -pe '($_)=/([0-9]+([.][0-9]+)+)/') + +# Download source code zip files from GitHub +curl "https://github.com/ghostery/ghostery-extension/archive/v$GHOSTERY_VERSION.zip" -O -J -L --fail +curl "https://github.com/cliqz-oss/browser-core/archive/v$CLIQZ_VERSION.zip" -O -J -L --fail + +# Make source-code zip +zip --quiet -R "$ZIP_FILE" "*" -x generate.sh *.DS_Store + +# Clean up +rm "browser-core-$CLIQZ_VERSION.zip" +rm "ghostery-extension-$GHOSTERY_VERSION.zip"