From ceaf003ef4a25c69d502cec14f2575d437be01de Mon Sep 17 00:00:00 2001 From: Jack Moffitt Date: Tue, 26 Sep 2017 14:50:49 -0500 Subject: [PATCH] Refactor builds to allow non-Servo builds on buildbot. --- buildbot/master/files/config/factories.py | 25 ++++--- buildbot/master/files/config/master.cfg | 84 ++++++++++++++--------- 2 files changed, 65 insertions(+), 44 deletions(-) diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index ef4b244e..8013a38f 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -43,19 +43,19 @@ def run(self): yield defer.returnValue(SUCCESS) -class ServoFactory(util.BuildFactory): +class GitUrlFactory(util.BuildFactory): """\ - Build factory which checks out the servo repo as the first build step. + Build factory which checks out a git url as the first build step. """ - def __init__(self, build_steps): + def __init__(self, url, build_steps): """\ Takes a list of Buildbot steps. Prefer using DynamicServoFactory to using this class directly. """ all_steps = [ steps.Git( - repourl=SERVO_REPO, + repourl=url, mode="full", method="fresh", retryFetch=True ), CheckRevisionStep(), @@ -67,8 +67,8 @@ def __init__(self, build_steps): class StepsYAMLParsingStep(buildstep.ShellMixin, buildstep.BuildStep): """\ - Step which reads the YAML steps configuration in the main servo repo - and dynamically adds test steps. + Step which reads the YAML steps configuration in the repo and dynamically + adds test steps. """ haltOnFailure = True @@ -246,25 +246,24 @@ def make_pkill_step(self, target): ) -class DynamicServoFactory(ServoFactory): +class DynamicGitUrlFactory(GitUrlFactory): """\ Smart factory which takes a list of shell commands - from a YAML file located in the main servo/servo repository + from a YAML file located in the repository and creates the appropriate Buildbot Steps. Uses heuristics to infer Step type, if there are any logfiles, etc. """ - def __init__(self, builder_name, environment): + def __init__(self, url, builder_name, environment, yaml_path): # util.BuildFactory is an old-style class so we cannot use super() # but must hardcode the superclass here - ServoFactory.__init__(self, [ - StepsYAMLParsingStep(builder_name, environment, - "etc/ci/buildbot_steps.yml") + ServoFactory.__init__(self, url, [ + StepsYAMLParsingStep(builder_name, environment, yaml_path) ]) -doc = ServoFactory([ +doc = GitUrlFactory(SERVO_REPO, [ # This is not dynamic because a) we need to pass the logEnviron kwarg # and b) changes to the documentation build are already encapsulated # in the upload_docs.sh script; any further changes should go through diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index c6075f75..3d99384c 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -58,6 +58,11 @@ def servo_master_filter(c): c.who.startswith('bors-servo') and c.branch == "master") +def repo_auto_try_filter(c): + return (c.project != 'servo/servo' and + c.who.startswith('bors-servo') and + c.branch in ["auto", "try"]) + c['schedulers'] = [] c['schedulers'].append(schedulers.AnyBranchScheduler( @@ -82,6 +87,15 @@ c['schedulers'].append(schedulers.AnyBranchScheduler( ], change_filter=util.ChangeFilter(filter_fn=servo_auto_try_filter), )) +c['schedulers'].append(schedulers.AnyBranchScheduler( + name="repo-auto", + treeStableTimer=None, + builderNames=[ + "linux-repo", + "mac-repo", + ], + change_filter=util.ChangeFilter(filter_fn=repo_auto_try_filter), +)) c['schedulers'].append(schedulers.SingleBranchScheduler( name="doc-push", treeStableTimer=None, @@ -145,49 +159,57 @@ def branch_priority(builder, requests): return requests[0] -class DynamicServoBuilder(util.BuilderConfig): +class DynamicRepoBuilder(util.BuilderConfig): """\ - Builder which uses DynamicServoFactory to run steps - from a YAML file in the main servo repo. + Builder which uses DynamicGitUrlFactory to run steps + from a YAML file in the repo. """ - def __init__(self, name, slavenames, environment): + def __init__(self, repo_owner, repo_name, name, slavenames, environment): # util.BuilderConfig is an old-style class so we cannot use super() # but must hardcode the superclass here util.BuilderConfig.__init__( self, name=name, slavenames=slavenames, - factory=factories.DynamicServoFactory(name, environment), + factory=factories.DynamicGitUrlFactory("https://github.com/%s/%s" % (repo_owner, repo_name), + name, environment, + "etc/ci/buildbot_steps.yml"), nextBuild=branch_priority, canStartBuild=util.enforceChosenSlave, + properties={ + "repo_owner": repo_owner, + "repo_name": repo_name, + }, ) c['builders'] = [ - DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), - DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), - DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32), - DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64), - DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-nightly", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-rel-css", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-rel-intermittent", LINUX_SLAVES, - envs.build_linux), - DynamicServoBuilder("linux-rel-nogate", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-rel-wpt", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-nightly", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-css1", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-css2", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-intermittent", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt1", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt2", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt3", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt4", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("windows-msvc-dev", WINDOWS_SLAVES, - envs.build_windows_msvc), - DynamicServoBuilder("windows-msvc-nightly", WINDOWS_SLAVES, - envs.build_windows_msvc), + DynamicRepoBuilder("servo", "servo", "android", CROSS_SLAVES, envs.build_android), + DynamicRepoBuilder("servo", "servo", "android-nightly", CROSS_SLAVES, envs.build_android), + DynamicRepoBuilder("servo", "servo", "arm32", CROSS_SLAVES, envs.build_arm32), + DynamicRepoBuilder("servo", "servo", "arm64", CROSS_SLAVES, envs.build_arm64), + DynamicRepoBuilder("servo", "servo", "linux-dev", LINUX_SLAVES, envs.build_linux), + DynamicRepoBuilder("servo", "servo", "linux-nightly", LINUX_SLAVES, envs.build_linux), + DynamicRepoBuilder("servo", "servo", "linux-rel-css", LINUX_SLAVES, envs.build_linux), + DynamicRepoBuilder("servo", "servo", "linux-rel-intermittent", LINUX_SLAVES, + envs.build_linux), + DynamicRepoBuilder("servo", "servo", "linux-rel-nogate", LINUX_SLAVES, envs.build_linux), + DynamicRepoBuilder("servo", "servo", "linux-rel-wpt", LINUX_SLAVES, envs.build_linux), + DynamicRepoBuilder("servo", "servo", "mac-dev-unit", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-nightly", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-css1", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-css2", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-intermittent", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-wpt1", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-wpt2", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-wpt3", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "mac-rel-wpt4", MAC_SLAVES, envs.build_mac), + DynamicRepoBuilder("servo", "servo", "windows-msvc-dev", WINDOWS_SLAVES, + envs.build_windows_msvc), + DynamicRepoBuilder("servo", "servo", "windows-msvc-nightly", WINDOWS_SLAVES, + envs.build_windows_msvc), + DyanmicRepoBuilder("servo", "webrender", "linux-repo", LINUX_SLAVES, envs.build_linux), + DynamicRepoBuilder("servo", "webrender", "mac-repo", MAC_SLAVES, envs.build_mac), # The below builders are not dynamic but rather have hard-coded factories util.BuilderConfig( name="doc", @@ -223,8 +245,8 @@ c['status'] = [ ), status.GitHubStatus( token=GITHUB_STATUS_TOKEN, - repoOwner='servo', - repoName='servo', + repoOwner=util.Interpolate("%(prop:repo_owner)s"), + repoName=util.Interpolate("%(prop:repo_name)s"), startDescription="Build started.", endDescription="Build done.", ),