From dffafa46fb2a2c11bf4c7aa03a4ab3cacc506ddc Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Mon, 16 Mar 2020 09:11:03 +0900 Subject: [PATCH 01/11] test codecommit --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 02ebee505a..7aa24c95d5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "vendor/depot_tools"] path = vendor/depot_tools - url = https://chromium.googlesource.com/chromium/tools/depot_tools.git + url = ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/depot_tools [submodule "vendor/jinja"] path = vendor/jinja - url = git://github.com/pallets/jinja.git + url = ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/jinja [submodule "vendor/gn-project-generators"] path = vendor/gn-project-generators - url = git://github.com/brave/gn-project-generators + url = ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/gn-project-generators From 39f9222ffcff10b0a09b05772d02ba4aadc28ebf Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Wed, 18 Mar 2020 10:25:59 +0900 Subject: [PATCH 02/11] update chromium to mirror --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0a22303d2a..2f3f5901e5 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "dir": "src", "tag": "80.0.3987.132", "repository": { - "url": "https://chromium.googlesource.com/chromium/src.git" + "url": "https://github.com/chromium/chromium.git" }, "custom_deps": { "src/third_party/WebKit/LayoutTests": "%None%", From ce68c8920734e6f702d3417c9c51500b505f2f99 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Thu, 19 Mar 2020 09:19:22 +0900 Subject: [PATCH 03/11] use codecommit as mirror --- Jenkinsfile | 254 +++++++++++++++++++++++++-------------------------- package.json | 2 +- 2 files changed, 128 insertions(+), 128 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3fd9f857cc..af9f19a9d7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -251,7 +251,7 @@ pipeline { beforeAgent true expression { !SKIP_LINUX } } - agent { label "linux-ci" } + agent { label "linux-ci-backup" } environment { GIT_CACHE_PATH = "${HOME}/cache" } @@ -332,59 +332,59 @@ pipeline { sh "npm run build -- ${BUILD_TYPE} --channel=${CHANNEL}" } } - stage("audit-network") { - when { - expression { RUN_NETWORK_AUDIT } - } - steps { - timeout(time: 4, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - sh "npm run network-audit -- --output_path=\"${OUT_DIR}/brave\"" - } - } - } - } - stage("test-unit") { - steps { - timeout(time: 60, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - sh "npm run test -- brave_unit_tests ${BUILD_TYPE} --output brave_unit_tests.xml" - xunit([GoogleTest(pattern: "src/brave_unit_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) - xunit([GoogleTest(pattern: "src/brave_installer_unittests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) - } - } - } - } - } - stage("test-browser") { - steps { - timeout(time: 20, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - sh "npm run test -- brave_browser_tests ${BUILD_TYPE} --output brave_browser_tests.xml" - xunit([GoogleTest(pattern: "src/brave_browser_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) - } - } - } - } - } - stage("dist") { - steps { - sh "npm run create_dist -- ${BUILD_TYPE} --channel=${CHANNEL}" - } - } - stage("s3-upload") { - steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - sh """ - cd ${OUT_DIR} - aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "brave-*.deb" - aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "brave-*.rpm" - """ - } - } - } + // stage("audit-network") { + // when { + // expression { RUN_NETWORK_AUDIT } + // } + // steps { + // timeout(time: 4, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // sh "npm run network-audit -- --output_path=\"${OUT_DIR}/brave\"" + // } + // } + // } + // } + // stage("test-unit") { + // steps { + // timeout(time: 60, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // script { + // sh "npm run test -- brave_unit_tests ${BUILD_TYPE} --output brave_unit_tests.xml" + // xunit([GoogleTest(pattern: "src/brave_unit_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) + // xunit([GoogleTest(pattern: "src/brave_installer_unittests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) + // } + // } + // } + // } + // } + // stage("test-browser") { + // steps { + // timeout(time: 20, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // script { + // sh "npm run test -- brave_browser_tests ${BUILD_TYPE} --output brave_browser_tests.xml" + // xunit([GoogleTest(pattern: "src/brave_browser_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) + // } + // } + // } + // } + // } + // stage("dist") { + // steps { + // sh "npm run create_dist -- ${BUILD_TYPE} --channel=${CHANNEL}" + // } + // } + // stage("s3-upload") { + // steps { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // sh """ + // cd ${OUT_DIR} + // aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "brave-*.deb" + // aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "brave-*.rpm" + // """ + // } + // } + // } } } stage("macos") { @@ -473,79 +473,79 @@ pipeline { sh "npm run build -- ${BUILD_TYPE} --channel=${CHANNEL} ${SKIP_SIGNING}" } } - stage("audit-network") { - when { - expression { RUN_NETWORK_AUDIT } - } - steps { - timeout(time: 4, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - sh "npm run network-audit -- --output_path=\"${OUT_DIR}/Brave\\ Browser${CHANNEL_CAPITALIZED_BACKSLASHED_SPACED}.app/Contents/MacOS/Brave\\ Browser${CHANNEL_CAPITALIZED_BACKSLASHED_SPACED}\"" - } - } - } - } - stage("test-unit") { - steps { - timeout(time: 60, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - sh "npm run test -- brave_unit_tests ${BUILD_TYPE} --output brave_unit_tests.xml" - xunit([GoogleTest(pattern: "src/brave_unit_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) - xunit([GoogleTest(pattern: "src/brave_installer_unittests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) - } - } - } - } - } - stage("test-browser") { - steps { - timeout(time: 20, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - sh "npm run test -- brave_browser_tests ${BUILD_TYPE} --output brave_browser_tests.xml" - xunit([GoogleTest(pattern: "src/brave_browser_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) - } - } - } - } - } - stage("dist") { - environment { - SIGN_WIDEVINE_CERT = credentials("widevine_brave_prod_cert.der") - SIGN_WIDEVINE_KEY = credentials("widevine_brave_prod_key.pem") - } - steps { - sh """ - security unlock-keychain -p "${KEYCHAIN_PASS}" "${KEYCHAIN_PATH}" - npm run create_dist -- ${BUILD_TYPE} --channel=${CHANNEL} ${SKIP_SIGNING} --mac_signing_keychain=${KEYCHAIN} --mac_signing_identifier=${MAC_APPLICATION_SIGNING_IDENTIFIER} --mac_installer_signing_identifier=${MAC_INSTALLER_SIGNING_IDENTIFIER} - security lock-keychain -a - """ - } - } - stage("test-install") { - steps { - timeout(time: 5, unit: "MINUTES") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - testInstallMac() - } - } - } - } - stage("s3-upload") { - steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - withAWS(credentials: "mac-build-s3-upload-artifacts", region: "us-west-2") { - sh """ - cd ${OUT_DIR} - aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "unsigned_dmg/Brave*.dmg" - aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "Brave*.dmg" - aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "Brave*.pkg" - """ - } - } - } - } + // stage("audit-network") { + // when { + // expression { RUN_NETWORK_AUDIT } + // } + // steps { + // timeout(time: 4, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // sh "npm run network-audit -- --output_path=\"${OUT_DIR}/Brave\\ Browser${CHANNEL_CAPITALIZED_BACKSLASHED_SPACED}.app/Contents/MacOS/Brave\\ Browser${CHANNEL_CAPITALIZED_BACKSLASHED_SPACED}\"" + // } + // } + // } + // } + // stage("test-unit") { + // steps { + // timeout(time: 60, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // script { + // sh "npm run test -- brave_unit_tests ${BUILD_TYPE} --output brave_unit_tests.xml" + // xunit([GoogleTest(pattern: "src/brave_unit_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) + // xunit([GoogleTest(pattern: "src/brave_installer_unittests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) + // } + // } + // } + // } + // } + // stage("test-browser") { + // steps { + // timeout(time: 20, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // script { + // sh "npm run test -- brave_browser_tests ${BUILD_TYPE} --output brave_browser_tests.xml" + // xunit([GoogleTest(pattern: "src/brave_browser_tests.xml", deleteOutputFiles: false, failIfNotNew: true, skipNoTestFiles: false, stopProcessingIfError: false)]) + // } + // } + // } + // } + // } + // stage("dist") { + // environment { + // SIGN_WIDEVINE_CERT = credentials("widevine_brave_prod_cert.der") + // SIGN_WIDEVINE_KEY = credentials("widevine_brave_prod_key.pem") + // } + // steps { + // sh """ + // security unlock-keychain -p "${KEYCHAIN_PASS}" "${KEYCHAIN_PATH}" + // npm run create_dist -- ${BUILD_TYPE} --channel=${CHANNEL} ${SKIP_SIGNING} --mac_signing_keychain=${KEYCHAIN} --mac_signing_identifier=${MAC_APPLICATION_SIGNING_IDENTIFIER} --mac_installer_signing_identifier=${MAC_INSTALLER_SIGNING_IDENTIFIER} + // security lock-keychain -a + // """ + // } + // } + // stage("test-install") { + // steps { + // timeout(time: 5, unit: "MINUTES") { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // testInstallMac() + // } + // } + // } + // } + // stage("s3-upload") { + // steps { + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // withAWS(credentials: "mac-build-s3-upload-artifacts", region: "us-west-2") { + // sh """ + // cd ${OUT_DIR} + // aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "unsigned_dmg/Brave*.dmg" + // aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "Brave*.dmg" + // aws s3 cp --no-progress . s3://${BRAVE_ARTIFACTS_S3_BUCKET}/${BUILD_TAG_SLASHED} --recursive --exclude="*" --include "Brave*.pkg" + // """ + // } + // } + // } + // } } } stage("windows-x64") { diff --git a/package.json b/package.json index 2f3f5901e5..349b41b035 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "dir": "src", "tag": "80.0.3987.132", "repository": { - "url": "https://github.com/chromium/chromium.git" + "url": "ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/chromium_src" }, "custom_deps": { "src/third_party/WebKit/LayoutTests": "%None%", From 2e73bb63f6c2d286c98c8d337a5973ba1be442b2 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Wed, 1 Apr 2020 17:10:39 +0900 Subject: [PATCH 04/11] test codecommit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7fa0df2e0c..73ab045faa 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "dir": "src", "tag": "81.0.4044.83", "repository": { - "url": "ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/chromium_src" + "url": "https://git-codecommit.us-west-2.amazonaws.com/v1/repos/chromium_src" }, "custom_deps": { "src/third_party/WebKit/LayoutTests": "%None%", From f8e940353f492407ccc55f81ac7442c17f3e7c60 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Fri, 3 Apr 2020 11:07:11 +0900 Subject: [PATCH 05/11] test commit mirror no history --- .gitmodules | 6 ++--- Jenkinsfile | 76 +++++++++++++++++++++++----------------------------- package.json | 3 +-- 3 files changed, 38 insertions(+), 47 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3b863fbb12..02ebee505a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "vendor/depot_tools"] path = vendor/depot_tools - url = https://git-codecommit.us-west-2.amazonaws.com/v1/repos/depot_tools + url = https://chromium.googlesource.com/chromium/tools/depot_tools.git [submodule "vendor/jinja"] path = vendor/jinja - url = https://git-codecommit.us-west-2.amazonaws.com/v1/repos/jinja + url = git://github.com/pallets/jinja.git [submodule "vendor/gn-project-generators"] path = vendor/gn-project-generators - url = https://git-codecommit.us-west-2.amazonaws.com/v1/repos/gn-project-generators + url = git://github.com/brave/gn-project-generators diff --git a/Jenkinsfile b/Jenkinsfile index 8a452bde2b..8963755bd0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,6 @@ pipeline { choice(name: "BUILD_TYPE", choices: ["Release", "Debug"], description: "") choice(name: "CHANNEL", choices: ["nightly", "dev", "beta", "release", "development"], description: "") string(name: "SLACK_BUILDS_CHANNEL", defaultValue: "#build-downloads-bot", description: "The Slack channel to send the list of artifact download links to. Leave blank to skip sending the message.") - string(name: "CHROMIUM_SRC", defaultValue: "https://github.com/chromium/chromium", description: "or use https://chromium.googlesource.com/chromium/src.git") booleanParam(name: "SKIP_SIGNING", defaultValue: true, description: "") booleanParam(name: "WIPE_WORKSPACE", defaultValue: false, description: "") booleanParam(name: "SKIP_INIT", defaultValue: false, description: "") @@ -98,12 +97,10 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - timeout(time: 1, unit: "HOURS") { - sh """ - rm -rf src/brave - npm run init -- --target_os=android - """ - } + sh """ + rm -rf src/brave + npm run init -- --target_os=android + """ } } stage("lint") { @@ -193,12 +190,10 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - timeout(time: 1, unit: "HOURS") { - sh """ - rm -rf src/brave - npm run init -- --target_os=ios - """ - } + sh """ + rm -rf src/brave + npm run init -- --target_os=ios + """ } } stage("lint") { @@ -291,12 +286,10 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - timeout(time: 1, unit: "HOURS") { - sh """ - rm -rf src/brave - npm run init - """ - } + sh """ + rm -rf src/brave + npm run init + """ } } stage("lint") { @@ -431,12 +424,10 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - timeout(time: 1, unit: "HOURS") { - sh """ - rm -rf src/brave - npm run init - """ - } + sh """ + rm -rf src/brave + npm run init + """ } } stage("lint") { @@ -553,7 +544,7 @@ pipeline { } agent { node { - label "windows-ci" + label "windows-ci-backup" customWorkspace "C:\\" + BRANCH[-4..-1] } } @@ -583,7 +574,7 @@ pipeline { \$ErrorActionPreference = "Stop" \$PSDefaultParameterValues['Out-File:Encoding'] = "utf8" jq "del(.config.projects[\\`"brave-core\\`"].branch) | .config.projects[\\`"brave-core\\`"].branch=\\`"${BRANCH}\\`"" package.json > package.json.new - jq '.config.projects.chrome.repository.url=\\`"${CHROMIUM_SRC}\\`"' package.json.new > package.json + Move-Item -Force package.json.new package.json """ } } @@ -612,15 +603,13 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - timeout(time: 1, unit: "HOURS") { - powershell """ - Remove-Item -Recurse -Force src/brave - git gc - git -C vendor/depot_tools clean -fxd - \$ErrorActionPreference = "Stop" - npm run init - """ - } + powershell """ + Remove-Item -Recurse -Force src/brave + git gc + git -C vendor/depot_tools clean -fxd + \$ErrorActionPreference = "Stop" + npm run init + """ } } stage("lint") { @@ -779,7 +768,7 @@ def setEnv() { env.BRANCH_PRODUCTIVITY_HOMEPAGE = "https://github.com/brave/brave-browser/pull/${bbPrNumber}" env.BRANCH_PRODUCTIVITY_NAME = "Brave Browser PR #${bbPrNumber}" env.BRANCH_PRODUCTIVITY_DESCRIPTION = bbPrDetails.title - env.BRANCH_PRODUCTIVITY_USER = bbPrDetails.user.login + env.BRANCH_PRODUCTIVITY_USER = env.SLACK_USERNAME ?: bbPrDetails.user.login } BRANCH_EXISTS_IN_BC = httpRequest(url: GITHUB_API + "/brave-core/branches/" + BRANCH, validResponseCodes: "100:499", customHeaders: [[name: "Authorization", value: "token ${PR_BUILDER_TOKEN}"]], quiet: !DEBUG).status.equals(200) if (BRANCH_EXISTS_IN_BC) { @@ -845,7 +834,11 @@ def sendSlackDownloadsNotification() { if (!attachments.isEmpty()) { def messageText = getSlackMessageText() echo "Sending builds message: '${messageText}'." - slackSend(channel: SLACK_BUILDS_CHANNEL, message: messageText, attachments: attachments) + slackSend( + channel: SLACK_BUILDS_CHANNEL, + message: messageText, + attachments: attachments + ) } else { echo "Not sending message, no files found." } @@ -859,7 +852,7 @@ def getSlackFileAttachments () { if (!file.directory && file.name != "build.txt") { attachments.add([ title: file.name, - title_link: "https://" + BRAVE_ARTIFACTS_S3_BUCKET + ".s3.amazonaws.com/" + BUILD_TAG_SLASHED.replace('%2F', '%252F') + "/" + file.path, + title_link: "https://" + BRAVE_ARTIFACTS_S3_BUCKET + ".s3.amazonaws.com/" + BUILD_TAG_SLASHED + "/" + file.path, footer: byteLengthToString(file.length) ]) } @@ -874,8 +867,7 @@ def getSlackMessageText () { } if (env.SLACK_USERNAME) { messageText += " by <${env.SLACK_USERNAME}>" - } - else if (env.BRANCH_PRODUCTIVITY_USER) { + } else if (env.BRANCH_PRODUCTIVITY_USER) { messageText += " by ${env.BRANCH_PRODUCTIVITY_USER}" } if (env.BRANCH_PRODUCTIVITY_DESCRIPTION) { @@ -909,7 +901,7 @@ def pin() { echo "Pinning brave-core locally to use branch ${BRANCH}" sh """ jq 'del(.config.projects["brave-core"].branch) | .config.projects["brave-core"].branch="${BRANCH}"' package.json > package.json.new - jq '.config.projects.chrome.repository.url="${CHROMIUM_SRC}"' package.json.new > package.json + mv package.json.new package.json """ } diff --git a/package.json b/package.json index d64443a996..11018a1bcd 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,8 @@ "projects": { "chrome": { "dir": "src", - "tag": "81.0.4044.83", "repository": { - "url": "https://git-codecommit.us-west-2.amazonaws.com/v1/repos/chromium_src" + "url": "ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/chromium-81.0.4044.83" }, "custom_deps": { "src/third_party/WebKit/LayoutTests": "%None%", From 67cc584822492420181b1925ff4f3a69a76636b0 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Mon, 6 Apr 2020 08:16:42 +0900 Subject: [PATCH 06/11] test commit mirror no history --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 11018a1bcd..947981df1f 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "chrome": { "dir": "src", "repository": { - "url": "ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/chromium-81.0.4044.83" + "url": "https://github.com/brave/chromium-81.0.4044.83.git" }, "custom_deps": { "src/third_party/WebKit/LayoutTests": "%None%", From cb9276f9817bbb4f750a264187b0d50d73497596 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Mon, 6 Apr 2020 09:41:11 +0900 Subject: [PATCH 07/11] test commit mirror no history --- lib/util.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/util.js b/lib/util.js index b97e1966b8..9b10221a41 100755 --- a/lib/util.js +++ b/lib/util.js @@ -532,8 +532,10 @@ const util = { util.run('git', ['-C', config.srcDir, 'remote', 'remove', 'origin'], options) util.run('git', ['-C', config.srcDir, 'remote', 'add', 'origin', config.chromiumRepo], options) console.log('setting src/brave remote `origin` to ' + config.braveCoreRepo) - util.run('git', ['-C', config.braveCoreDir, 'remote', 'remove', 'origin'], options) - util.run('git', ['-C', config.braveCoreDir, 'remote', 'add', 'origin', config.braveCoreRepo], options) + if (fs.existsSync(config.braveCoreDir)) { + util.run('git', ['-C', config.braveCoreDir, 'remote', 'remove', 'origin'], options) + util.run('git', ['-C', config.braveCoreDir, 'remote', 'add', 'origin', config.braveCoreRepo], options) + } } let args = ['sync', '--force', '--nohooks', '--with_branch_heads', '--with_tags'] From 176cda1d6acc9cd39ffc73b2fb065a0e08a13919 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Mon, 6 Apr 2020 11:24:51 +0900 Subject: [PATCH 08/11] test commit mirror no history --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 947981df1f..9c662224b3 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "projects": { "chrome": { "dir": "src", + "branch": "81.0.4044.83", "repository": { "url": "https://github.com/brave/chromium-81.0.4044.83.git" }, From a452ac8c78b46176756f7538323e582cf6bf39b2 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Mon, 6 Apr 2020 15:49:29 +0900 Subject: [PATCH 09/11] test commit mirror no history --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9c662224b3..4102dc099f 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "projects": { "chrome": { "dir": "src", - "branch": "81.0.4044.83", + "tag": "81.0.4044.83", "repository": { "url": "https://github.com/brave/chromium-81.0.4044.83.git" }, From 4ea57df6114310b1a5c4492197e4c18311f718e7 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Wed, 8 Apr 2020 11:08:57 +0900 Subject: [PATCH 10/11] test windows ci build on spot --- Jenkinsfile | 119 ++++++++++++++++++++++++++-------------------------- 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8963755bd0..af657c7f7d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,7 @@ pipeline { choice(name: "BUILD_TYPE", choices: ["Release", "Debug"], description: "") choice(name: "CHANNEL", choices: ["nightly", "dev", "beta", "release", "development"], description: "") string(name: "SLACK_BUILDS_CHANNEL", defaultValue: "#build-downloads-bot", description: "The Slack channel to send the list of artifact download links to. Leave blank to skip sending the message.") + string(name: "CHROMIUM_SRC", defaultValue: "https://github.com/chromium/chromium", description: "or use https://chromium.googlesource.com/chromium/src.git") booleanParam(name: "SKIP_SIGNING", defaultValue: true, description: "") booleanParam(name: "WIPE_WORKSPACE", defaultValue: false, description: "") booleanParam(name: "SKIP_INIT", defaultValue: false, description: "") @@ -73,9 +74,6 @@ pipeline { } } stage("pin") { - when { - expression { BRANCH_EXISTS_IN_BC } - } steps { pin() } @@ -97,10 +95,12 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - sh """ - rm -rf src/brave - npm run init -- --target_os=android - """ + timeout(time: 1, unit: "HOURS") { + sh """ + rm -rf src/brave + npm run init -- --target_os=android + """ + } } } stage("lint") { @@ -166,9 +166,6 @@ pipeline { } } stage("pin") { - when { - expression { BRANCH_EXISTS_IN_BC } - } steps { pin() } @@ -190,10 +187,12 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - sh """ - rm -rf src/brave - npm run init -- --target_os=ios - """ + timeout(time: 1, unit: "HOURS") { + sh """ + rm -rf src/brave + npm run init -- --target_os=ios + """ + } } } stage("lint") { @@ -262,9 +261,6 @@ pipeline { } } stage("pin") { - when { - expression { BRANCH_EXISTS_IN_BC } - } steps { pin() } @@ -286,10 +282,12 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - sh """ - rm -rf src/brave - npm run init - """ + timeout(time: 1, unit: "HOURS") { + sh """ + rm -rf src/brave + npm run init + """ + } } } stage("lint") { @@ -396,9 +394,6 @@ pipeline { } } stage("pin") { - when { - expression { BRANCH_EXISTS_IN_BC } - } steps { pin() } @@ -424,10 +419,12 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - sh """ - rm -rf src/brave - npm run init - """ + timeout(time: 1, unit: "HOURS") { + sh """ + rm -rf src/brave + npm run init + """ + } } } stage("lint") { @@ -565,17 +562,10 @@ pipeline { } } stage("pin") { - when { - expression { BRANCH_EXISTS_IN_BC } - } steps { - echo "Pinning brave-core locally to use branch ${BRANCH}" - powershell """ - \$ErrorActionPreference = "Stop" - \$PSDefaultParameterValues['Out-File:Encoding'] = "utf8" - jq "del(.config.projects[\\`"brave-core\\`"].branch) | .config.projects[\\`"brave-core\\`"].branch=\\`"${BRANCH}\\`"" package.json > package.json.new - Move-Item -Force package.json.new package.json - """ + script { + pinWindows() + } } } stage("install") { @@ -603,13 +593,15 @@ pipeline { expression { return !fileExists("src/brave/package.json") || !SKIP_INIT } } steps { - powershell """ - Remove-Item -Recurse -Force src/brave - git gc - git -C vendor/depot_tools clean -fxd - \$ErrorActionPreference = "Stop" - npm run init - """ + timeout(time: 1, unit: "HOURS") { + powershell """ + Remove-Item -Recurse -Force src/brave + git gc + git -C vendor/depot_tools clean -fxd + \$ErrorActionPreference = "Stop" + npm run init + """ + } } } stage("lint") { @@ -730,6 +722,7 @@ def setEnv() { BUILD_TYPE = params.BUILD_TYPE CHANNEL = params.CHANNEL SLACK_BUILDS_CHANNEL = params.SLACK_BUILDS_CHANNEL + CHROMIUM_SRC = params.CHROMIUM_SRC DCHECK_ALWAYS_ON = params.DCHECK_ALWAYS_ON CHANNEL_CAPITALIZED = CHANNEL.equals("release") ? "" : CHANNEL.capitalize() CHANNEL_CAPITALIZED_BACKSLASHED_SPACED = CHANNEL.equals("release") ? "" : "\\ " + CHANNEL.capitalize() @@ -752,9 +745,11 @@ def setEnv() { RUN_NETWORK_AUDIT = false BRANCH = env.BRANCH_NAME BASE_BRANCH = "master" + PIN_BRANCH = "master" if (env.CHANGE_BRANCH) { BRANCH = env.CHANGE_BRANCH BASE_BRANCH = env.CHANGE_TARGET + PIN_BRANCH = readJSON(text: httpRequest(url: "https://raw.githubusercontent.com/brave/brave-browser/${BRANCH}/package.json").content).config.projects["brave-core"].branch def bbPrNumber = readJSON(text: httpRequest(url: GITHUB_API + "/brave-browser/pulls?head=brave:" + BRANCH, customHeaders: [[name: "Authorization", value: "token ${PR_BUILDER_TOKEN}"]], quiet: !DEBUG).content)[0].number def bbPrDetails = readJSON(text: httpRequest(url: GITHUB_API + "/brave-browser/pulls/" + bbPrNumber, customHeaders: [[name: "Authorization", value: "token ${PR_BUILDER_TOKEN}"]], quiet: !DEBUG).content) SKIP = bbPrDetails.mergeable_state.equals("draft") || bbPrDetails.labels.count { label -> label.name.equalsIgnoreCase("CI/skip") }.equals(1) @@ -768,7 +763,7 @@ def setEnv() { env.BRANCH_PRODUCTIVITY_HOMEPAGE = "https://github.com/brave/brave-browser/pull/${bbPrNumber}" env.BRANCH_PRODUCTIVITY_NAME = "Brave Browser PR #${bbPrNumber}" env.BRANCH_PRODUCTIVITY_DESCRIPTION = bbPrDetails.title - env.BRANCH_PRODUCTIVITY_USER = env.SLACK_USERNAME ?: bbPrDetails.user.login + env.BRANCH_PRODUCTIVITY_USER = bbPrDetails.user.login } BRANCH_EXISTS_IN_BC = httpRequest(url: GITHUB_API + "/brave-core/branches/" + BRANCH, validResponseCodes: "100:499", customHeaders: [[name: "Authorization", value: "token ${PR_BUILDER_TOKEN}"]], quiet: !DEBUG).status.equals(200) if (BRANCH_EXISTS_IN_BC) { @@ -777,6 +772,7 @@ def setEnv() { env.BC_PR_NUMBER = bcPrDetails.number bcPrDetails = readJSON(text: httpRequest(url: GITHUB_API + "/brave-core/pulls/" + env.BC_PR_NUMBER, customHeaders: [[name: "Authorization", value: "token ${PR_BUILDER_TOKEN}"]], quiet: !DEBUG).content) BASE_BRANCH = bcPrDetails.base.ref + PIN_BRANCH = BRANCH SKIP = bcPrDetails.mergeable_state.equals("draft") || bcPrDetails.labels.count { label -> label.name.equalsIgnoreCase("CI/skip") }.equals(1) SKIP_ANDROID = SKIP_ANDROID || bcPrDetails.labels.count { label -> label.name.equalsIgnoreCase("CI/skip-android") }.equals(1) SKIP_IOS = SKIP_IOS || bcPrDetails.labels.count { label -> label.name.equalsIgnoreCase("CI/skip-ios") }.equals(1) @@ -834,11 +830,7 @@ def sendSlackDownloadsNotification() { if (!attachments.isEmpty()) { def messageText = getSlackMessageText() echo "Sending builds message: '${messageText}'." - slackSend( - channel: SLACK_BUILDS_CHANNEL, - message: messageText, - attachments: attachments - ) + slackSend(channel: SLACK_BUILDS_CHANNEL, message: messageText, attachments: attachments) } else { echo "Not sending message, no files found." } @@ -852,7 +844,7 @@ def getSlackFileAttachments () { if (!file.directory && file.name != "build.txt") { attachments.add([ title: file.name, - title_link: "https://" + BRAVE_ARTIFACTS_S3_BUCKET + ".s3.amazonaws.com/" + BUILD_TAG_SLASHED + "/" + file.path, + title_link: "https://" + BRAVE_ARTIFACTS_S3_BUCKET + ".s3.amazonaws.com/" + BUILD_TAG_SLASHED.replace('%2F', '%252F') + "/" + file.path, footer: byteLengthToString(file.length) ]) } @@ -867,7 +859,8 @@ def getSlackMessageText () { } if (env.SLACK_USERNAME) { messageText += " by <${env.SLACK_USERNAME}>" - } else if (env.BRANCH_PRODUCTIVITY_USER) { + } + else if (env.BRANCH_PRODUCTIVITY_USER) { messageText += " by ${env.BRANCH_PRODUCTIVITY_USER}" } if (env.BRANCH_PRODUCTIVITY_DESCRIPTION) { @@ -898,10 +891,20 @@ def getBuilds() { } def pin() { - echo "Pinning brave-core locally to use branch ${BRANCH}" + echo "Pinning brave-core locally to use branch ${PIN_BRANCH}" sh """ - jq 'del(.config.projects["brave-core"].branch) | .config.projects["brave-core"].branch="${BRANCH}"' package.json > package.json.new - mv package.json.new package.json + jq 'del(.config.projects["brave-core"].branch) | .config.projects["brave-core"].branch="${PIN_BRANCH}"' package.json > package.json.new + jq '.config.projects.chrome.repository.url="${CHROMIUM_SRC}"' package.json.new > package.json + """ +} + +def pinWindows() { + echo "Pinning brave-core locally to use branch ${PIN_BRANCH}" + powershell """ + \$ErrorActionPreference = "Stop" + \$PSDefaultParameterValues['Out-File:Encoding'] = "utf8" + jq "del(.config.projects[\\`"brave-core\\`"].branch) | .config.projects[\\`"brave-core\\`"].branch=\\`"${PIN_BRANCH}\\`"" package.json > package.json.new + jq ".config.projects.chrome.repository.url=\\`"${CHROMIUM_SRC}\\`"" package.json.new > package.json """ } @@ -973,10 +976,6 @@ def installWindows() { Get-ChildItem "Cert:\\LocalMachine\\My" | Remove-Item \$ErrorActionPreference = "Stop" npm install --no-optional - Copy-Item "${SOURCE_KEY_CER_PATH}" -Destination "${KEY_CER_PATH}" - Copy-Item "${SOURCE_KEY_PFX_PATH}" -Destination "${KEY_PFX_PATH}" - Import-Certificate -FilePath "${SIGN_WIDEVINE_CERT}" -CertStoreLocation "Cert:\\LocalMachine\\My" - Import-PfxCertificate -FilePath "${KEY_PFX_PATH}" -CertStoreLocation "Cert:\\LocalMachine\\My" -Password (ConvertTo-SecureString -String "${AUTHENTICODE_PASSWORD_UNESCAPED}" -AsPlaintext -Force) New-Item -Force -ItemType directory -Path "src\\third_party\\widevine\\scripts" Copy-Item "C:\\jenkins\\signature_generator.py" -Destination "src\\third_party\\widevine\\scripts\\" """ From 2d17de3c92011c01ac1d53a18eb47c79efa7f530 Mon Sep 17 00:00:00 2001 From: Linh Kikuchi Date: Wed, 13 May 2020 12:16:49 +0900 Subject: [PATCH 11/11] test to add android test-unit --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b454898d2f..243ec7e37b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -160,7 +160,7 @@ def startBraveBrowserBuild() { credentials('brave-builds-github-token-for-pr-builder') github('brave/devops', 'https') } - branch('master') + branch('android-ci-test-unit') } } scriptPath('jenkins/jobs/browser/Jenkinsfile')