From 82d8b4b9a040895c400070de0091ada0293ea53a Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Thu, 5 Apr 2018 14:50:40 +0200 Subject: [PATCH 01/11] WIP nightly builds on jenkins --- Dockerfile | 12 ++++++++++++ Jenkinsfile | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 Dockerfile create mode 100644 Jenkinsfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..d2217dc09 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM node:6.14.1 + +RUN npm install -g web-ext + +ARG UID +ARG GID +RUN groupadd jenkins -g $GID \ + && useradd -ms /bin/bash jenkins -u $UID -g $GID + +USER jenkins +COPY package.json /home/jenkins/ +RUN cd /home/jenkins/ && npm install diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..9a3763adb --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,48 @@ +node('docker') { + stage ('Checkout') { + checkout scm + } + + def img + + stage('Build Docker Image') { + img = docker.build('ghostery/build', '--build-arg UID=`id -u` --build-arg GID=`id -g` .') + } + + stage('Build Extension') { + img.inside() { + withCache { + // rerun postinstall for vendor-copy + sh 'npm run postinstall' + sh 'npm run build.prod' + sh 'web-ext build --overwrite-dest' + } + } + } + + stage('Publish') { + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: '06ec4a34-9d01-46df-9ff8-64c79eda8b14', + passwordVariable: 'AWS_SECRET_ACCESS_KEY', + usernameVariable: 'AWS_ACCESS_KEY_ID']]) { + sh 'aws s3 sync web-ext-artifacts/ s3://cdncliqz/update/ghostery/nightly_test/ --acl public-read' + } + } +} + +def withCache(Closure body=null) { + def cleanCache = { + sh 'rm -fr node_modules' + } + + try { + cleanCache() + // Main dependencies + sh 'cp -fr /home/jenkins/node_modules .' + + body() + } finally { + cleanCache() + } +} From ebdfaa03983676396bf649cb383d2db3a68e7c52 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Fri, 6 Apr 2018 09:52:50 +0200 Subject: [PATCH 02/11] Publish with build number. --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9a3763adb..f8469223f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,7 +26,8 @@ node('docker') { credentialsId: '06ec4a34-9d01-46df-9ff8-64c79eda8b14', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID']]) { - sh 'aws s3 sync web-ext-artifacts/ s3://cdncliqz/update/ghostery/nightly_test/ --acl public-read' + echo "${env.BRANCH_NAME}/${env.BUILD_NUMBER}" + sh "aws s3 sync web-ext-artifacts/ s3://cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/ --acl public-read" } } } From 7718afe0e538cd0501d2949987a5587333d53109 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Fri, 6 Apr 2018 09:54:06 +0200 Subject: [PATCH 03/11] Use package-lock.json in docker. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index d2217dc09..443603dcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,5 @@ RUN groupadd jenkins -g $GID \ USER jenkins COPY package.json /home/jenkins/ +COPY package-lock.json /home/jenkins/ RUN cd /home/jenkins/ && npm install From 924458af2c80fbe933972e6355a7d2aac3ea1539 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Fri, 6 Apr 2018 15:11:37 +0200 Subject: [PATCH 04/11] Test moab makezip --- Dockerfile | 4 +++- Jenkinsfile | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 443603dcd..4f14435fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM node:6.14.1 -RUN npm install -g web-ext +#RUN npm install -g web-ext +RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_2 > /bin/moab && \ + chmod +x /bin/moab ARG UID ARG GID diff --git a/Jenkinsfile b/Jenkinsfile index f8469223f..b4edf8aea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,6 +4,7 @@ node('docker') { } def img + def artifact stage('Build Docker Image') { img = docker.build('ghostery/build', '--build-arg UID=`id -u` --build-arg GID=`id -g` .') @@ -12,10 +13,10 @@ node('docker') { stage('Build Extension') { img.inside() { withCache { - // rerun postinstall for vendor-copy - sh 'npm run postinstall' - sh 'npm run build.prod' - sh 'web-ext build --overwrite-dest' + sh 'rm -r build/' + sh 'moab makezip' + // get the name of the firefox build + artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() } } } @@ -27,7 +28,9 @@ node('docker') { passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID']]) { echo "${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - sh "aws s3 sync web-ext-artifacts/ s3://cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/ --acl public-read" + def uploadLocation = "s3://cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${artifact}" + currentBuild.description = uploadLocation + sh "aws s3 cp build/${artifact} ${uploadLocation} --acl public-read" } } } From cfc2f0f2fa2ab27f8b1f78c41f46dbd9c384c228 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Fri, 6 Apr 2018 16:11:17 +0200 Subject: [PATCH 05/11] WIP fetch up-to-date libraries for build. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4f14435fa..55b418be4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM node:6.14.1 #RUN npm install -g web-ext -RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_2 > /bin/moab && \ +RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_3 > /bin/moab && \ chmod +x /bin/moab ARG UID From 6db0d29b4b8fcd04198928a8698ef0f7ce311156 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Fri, 6 Apr 2018 16:58:34 +0200 Subject: [PATCH 06/11] Add option to build with browser-core master --- Dockerfile | 4 +-- Jenkinsfile | 70 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 55b418be4..d5524bcad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM node:6.14.1 +FROM node:8 #RUN npm install -g web-ext -RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_3 > /bin/moab && \ +RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_2 > /bin/moab && \ chmod +x /bin/moab ARG UID diff --git a/Jenkinsfile b/Jenkinsfile index b4edf8aea..f20cc3f5a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,10 @@ + +properties([ + parameters([ + booleanParam(name: 'WITH_CLIQZ_MASTER', defaultValue: false, description: 'Builds with latest Cliqz master') + ]) +]) + node('docker') { stage ('Checkout') { checkout scm @@ -11,42 +18,45 @@ node('docker') { } stage('Build Extension') { - img.inside() { - withCache { - sh 'rm -r build/' - sh 'moab makezip' - // get the name of the firefox build - artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() + img.inside() { + withCache { + sh 'rm -r build' + if (params.WITH_CLIQZ_MASTER) { + sh 'npm install --save https://s3.amazonaws.com/cdncliqz/update/edge/ghostery/master/latest.tgz' + } + sh 'moab makezip' + // get the name of the firefox build + artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() + } } - } } stage('Publish') { - withCredentials([[ - $class: 'UsernamePasswordMultiBinding', - credentialsId: '06ec4a34-9d01-46df-9ff8-64c79eda8b14', - passwordVariable: 'AWS_SECRET_ACCESS_KEY', - usernameVariable: 'AWS_ACCESS_KEY_ID']]) { - echo "${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - def uploadLocation = "s3://cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${artifact}" - currentBuild.description = uploadLocation - sh "aws s3 cp build/${artifact} ${uploadLocation} --acl public-read" - } + withCredentials([[ + $class: 'UsernamePasswordMultiBinding', + credentialsId: '06ec4a34-9d01-46df-9ff8-64c79eda8b14', + passwordVariable: 'AWS_SECRET_ACCESS_KEY', + usernameVariable: 'AWS_ACCESS_KEY_ID']]) { + echo "${env.BRANCH_NAME}/${env.BUILD_NUMBER}" + def uploadLocation = "s3://cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${artifact}" + currentBuild.description = uploadLocation + sh "aws s3 cp build/${artifact} ${uploadLocation} --acl public-read" + } } } def withCache(Closure body=null) { - def cleanCache = { - sh 'rm -fr node_modules' - } - - try { - cleanCache() - // Main dependencies - sh 'cp -fr /home/jenkins/node_modules .' - - body() - } finally { - cleanCache() - } + def cleanCache = { + sh 'rm -fr node_modules' + } + + try { + cleanCache() + // Main dependencies + sh 'cp -fr /home/jenkins/node_modules .' + + body() + } finally { + cleanCache() + } } From 15e9f788be2234b25f64d3bb0422c49428b0389f Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Fri, 6 Apr 2018 17:42:45 +0200 Subject: [PATCH 07/11] Add signing step. --- Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f20cc3f5a..6ba2b1c49 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,6 +12,7 @@ node('docker') { def img def artifact + def uploadPath = "cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" stage('Build Docker Image') { img = docker.build('ghostery/build', '--build-arg UID=`id -u` --build-arg GID=`id -g` .') @@ -38,11 +39,16 @@ node('docker') { passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID']]) { echo "${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - def uploadLocation = "s3://cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/${artifact}" + def uploadLocation = "s3://${uploadPath}/${artifact}" currentBuild.description = uploadLocation sh "aws s3 cp build/${artifact} ${uploadLocation} --acl public-read" } } + + stage('Sign and Publish') { + def artifactUrl = "https://s3.amazonaws.com/${uploadPath}/${artifact}" + build job: 'addon-repack', parameters: [string(name: 'XPI_URL', value: artifactUrl), string(name: 'XPI_SIGN_CREDENTIALS', value: '41572f9c-06aa-46f0-9c3b-b7f4f78e9caa'), string(name: 'XPI_SIGN_REPO_URL', value: 'git@github.com:cliqz/xpi-sign.git')] + } } def withCache(Closure body=null) { From 24d8cbcb7ba3c9dd4864f1c053430251e4d3f051 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Mon, 9 Apr 2018 16:44:21 +0200 Subject: [PATCH 08/11] Make browser-core log. --- Dockerfile | 1 - Jenkinsfile | 24 +++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5524bcad..46b030084 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM node:8 -#RUN npm install -g web-ext RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_2 > /bin/moab && \ chmod +x /bin/moab diff --git a/Jenkinsfile b/Jenkinsfile index 6ba2b1c49..b55c6ef0e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,15 +20,17 @@ node('docker') { stage('Build Extension') { img.inside() { - withCache { - sh 'rm -r build' - if (params.WITH_CLIQZ_MASTER) { - sh 'npm install --save https://s3.amazonaws.com/cdncliqz/update/edge/ghostery/master/latest.tgz' - } - sh 'moab makezip' - // get the name of the firefox build - artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() + withCache { + sh 'rm -rf build' + if (params.WITH_CLIQZ_MASTER) { + sh 'npm install --save https://s3.amazonaws.com/cdncliqz/update/edge/ghostery/master/latest.tgz' } + // make browser-core noisy + sh 'sed -i \'s/global.__DEV__/true/1\' node_modules/browser-core/build/core/console.js' + sh 'moab makezip' + // get the name of the firefox build + artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() + } } } @@ -47,7 +49,11 @@ node('docker') { stage('Sign and Publish') { def artifactUrl = "https://s3.amazonaws.com/${uploadPath}/${artifact}" - build job: 'addon-repack', parameters: [string(name: 'XPI_URL', value: artifactUrl), string(name: 'XPI_SIGN_CREDENTIALS', value: '41572f9c-06aa-46f0-9c3b-b7f4f78e9caa'), string(name: 'XPI_SIGN_REPO_URL', value: 'git@github.com:cliqz/xpi-sign.git')] + build job: 'addon-repack', parameters: [ + string(name: 'XPI_URL', value: artifactUrl), + string(name: 'XPI_SIGN_CREDENTIALS', value: '41572f9c-06aa-46f0-9c3b-b7f4f78e9caa'), + string(name: 'XPI_SIGN_REPO_URL', value: 'git@github.com:cliqz/xpi-sign.git') + ] } } From 07a75692a44f60f0d86086dc364a36c1386ff889 Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Tue, 10 Apr 2018 10:05:01 +0200 Subject: [PATCH 09/11] Load github credentials to enable moab checkout of ghostery-libraries. --- Dockerfile | 2 +- Jenkinsfile | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 46b030084..b1fa910fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM node:8 -RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_2 > /bin/moab && \ +RUN curl https://s3.amazonaws.com/cdncliqz/update/ghostery/moab/moab_8319dab > /bin/moab && \ chmod +x /bin/moab ARG UID diff --git a/Jenkinsfile b/Jenkinsfile index b55c6ef0e..c2e8547b0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,9 @@ node('docker') { } // make browser-core noisy sh 'sed -i \'s/global.__DEV__/true/1\' node_modules/browser-core/build/core/console.js' - sh 'moab makezip' + withGithubCredentials { + sh 'moab makezip' + } // get the name of the firefox build artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() } @@ -72,3 +74,26 @@ def withCache(Closure body=null) { cleanCache() } } + +def withGithubCredentials(Closure body) { + withCredentials([sshUserPrivateKey( + credentialsId: '6739a36f-0b19-4f4d-b6e4-b01d0bc2e175', + keyFileVariable: 'GHOSTERY_CI_SSH_KEY') + ]) { + // initialise git+ssh access using cliqz-ci credentials + try { + sh '''#!/bin/bash -l + set -x + set -e + mkdir -p ~/.ssh + cp $GHOSTERY_CI_SSH_KEY ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + ''' + body() + } finally { + sh 'rm -f ~/.ssh/id_rsa' + sh 'rm -f ~/.ssh/known_hosts' + } + } +} \ No newline at end of file From af79226cdda9377a8a9300bda1ff220fb24891ab Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Tue, 10 Apr 2018 12:26:51 +0200 Subject: [PATCH 10/11] Tidy deployment. --- Jenkinsfile | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c2e8547b0..08d5f7e20 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ node('docker') { def img def artifact - def uploadPath = "cdncliqz/update/ghostery/nightly_test/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" + def uploadPath = "cdncliqz/update/ghostery/${env.BRANCH_NAME}" stage('Build Docker Image') { img = docker.build('ghostery/build', '--build-arg UID=`id -u` --build-arg GID=`id -g` .') @@ -49,13 +49,15 @@ node('docker') { } } - stage('Sign and Publish') { - def artifactUrl = "https://s3.amazonaws.com/${uploadPath}/${artifact}" - build job: 'addon-repack', parameters: [ - string(name: 'XPI_URL', value: artifactUrl), - string(name: 'XPI_SIGN_CREDENTIALS', value: '41572f9c-06aa-46f0-9c3b-b7f4f78e9caa'), - string(name: 'XPI_SIGN_REPO_URL', value: 'git@github.com:cliqz/xpi-sign.git') - ] + if (env.BRANCH_NAME == 'develop') { + stage('Sign and Publish') { + def artifactUrl = "https://s3.amazonaws.com/${uploadPath}/${artifact}" + build job: 'addon-repack', parameters: [ + string(name: 'XPI_URL', value: artifactUrl), + string(name: 'XPI_SIGN_CREDENTIALS', value: '41572f9c-06aa-46f0-9c3b-b7f4f78e9caa'), + string(name: 'XPI_SIGN_REPO_URL', value: 'git@github.com:cliqz/xpi-sign.git') + ] + } } } @@ -80,7 +82,7 @@ def withGithubCredentials(Closure body) { credentialsId: '6739a36f-0b19-4f4d-b6e4-b01d0bc2e175', keyFileVariable: 'GHOSTERY_CI_SSH_KEY') ]) { - // initialise git+ssh access using cliqz-ci credentials + // initialise git+ssh access using ghostery-ci credentials try { sh '''#!/bin/bash -l set -x From e071f77b3b508819c6c9bc2e9cafaa5ec9d1abad Mon Sep 17 00:00:00 2001 From: Sam Macbeth Date: Tue, 10 Apr 2018 17:24:26 +0200 Subject: [PATCH 11/11] Add Chrome packaging step. --- Jenkinsfile | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 08d5f7e20..47654ddc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ node('docker') { } def img - def artifact + def artifacts = [] def uploadPath = "cdncliqz/update/ghostery/${env.BRANCH_NAME}" stage('Build Docker Image') { @@ -31,21 +31,40 @@ node('docker') { sh 'moab makezip' } // get the name of the firefox build - artifact = sh(returnStdout: true, script: 'ls build/ | grep firefox').trim() + artifacts.add(sh(returnStdout: true, script: 'ls build/ | grep firefox').trim()) } } } - stage('Publish') { + stage('Package Chrome') { + withGithubCredentials { + def chromeArtifact = sh(returnStdout: true, script: 'ls build/ | grep chrome').trim().replace('.zip', '') + echo "${chromeArtifact}" + sh """#!/bin/bash -l + set -x + set -e + rm -rf ${chromeArtifact}/ + mkdir -p ${chromeArtifact} + unzip build/${chromeArtifact}.zip -d ${chromeArtifact} + tools/crxmake.sh ${chromeArtifact}/ ~/.ssh/id_rsa + mv ${chromeArtifact}.crx build/ + """ + artifacts.add("${chromeArtifact}.crx") + } + } + + stage('Publish Builds') { withCredentials([[ $class: 'UsernamePasswordMultiBinding', credentialsId: '06ec4a34-9d01-46df-9ff8-64c79eda8b14', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID']]) { echo "${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - def uploadLocation = "s3://${uploadPath}/${artifact}" + def uploadLocation = "s3://${uploadPath}/" currentBuild.description = uploadLocation - sh "aws s3 cp build/${artifact} ${uploadLocation} --acl public-read" + artifacts.each { + sh "aws s3 cp build/${it} ${uploadLocation} --acl public-read" + } } }