From 46198df7484b20298243b19753d7a6e1c9cedbe8 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Thu, 28 Sep 2017 18:06:25 -0400 Subject: [PATCH 1/2] Infer Buildbot workers from environment Use a dictionary for the lookup. Make the Environment class (mostly) immutable and hashable to allow using environment as dictionary keys. --- buildbot/master/files/config/environments.py | 23 +++++- buildbot/master/files/config/master.cfg | 77 +++++++++++--------- 2 files changed, 62 insertions(+), 38 deletions(-) diff --git a/buildbot/master/files/config/environments.py b/buildbot/master/files/config/environments.py index 41dc2901..9e6fbf34 100644 --- a/buildbot/master/files/config/environments.py +++ b/buildbot/master/files/config/environments.py @@ -6,6 +6,7 @@ class Environment(dict): """ Wrapper that allows 'adding' environment dictionaries to make it easy to build up environments piece by piece. + Also (mostly) immutable and hashable. """ def __init__(self, *args, **kwargs): @@ -17,27 +18,41 @@ def __init__(self, *args, **kwargs): assert type(k) == str assert type(self[k]) == str + def __hash__(self): + try: + return self._cached_hash + except AttributeError: + self._cached_hash = hash(tuple(sorted(self.items()))) + return self._cached_hash + + @classmethod + def _blocked_attribute(cls, *args, **kwargs): + raise TypeError('%r objects are immutable' % cls.__name__) + + __delitem__ = __setitem__ = _blocked_attribute + clear = pop = popitem = setdefault = update = _blocked_attribute + def copy(self): # Return an environment, not a plain dict return Environment(self) def __add__(self, other): assert type(self) == type(other) - combined = self.copy() + combined = dict(self.copy()) combined.update(other) # other takes precedence over self - return combined + return Environment(combined) def without(self, to_unset): """ Return a new Environment that does not contain the environment variables specified in the list of strings to_unset. """ - modified = self.copy() + modified = dict(self.copy()) assert type(to_unset) == list for env_var in to_unset: if env_var in modified: modified.pop(env_var) - return modified + return Environment(modified) doc = Environment({ diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index c6075f75..ceb6cb8c 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -9,10 +9,10 @@ from passwords import SLAVE_PASSWORD, CHANGE_PASSWORD from passwords import HOMU_BUILDBOT_SECRET, GITHUB_STATUS_TOKEN -LINUX_SLAVES = ["servo-linux{}".format(i) for i in range(1, 7)] -MAC_SLAVES = ["servo-mac{}".format(i) for i in range(1, 10)] -CROSS_SLAVES = ["servo-linux-cross{}".format(i) for i in range(1, 4)] -WINDOWS_SLAVES = ["servo-windows{}".format(i) for i in range(1, 3)] +LINUX_WORKERS = ["servo-linux{}".format(i) for i in range(1, 7)] +MAC_WORKERS = ["servo-mac{}".format(i) for i in range(1, 10)] +CROSS_WORKERS = ["servo-linux-cross{}".format(i) for i in range(1, 4)] +WINDOWS_WORKERS = ["servo-windows{}".format(i) for i in range(1, 3)] c = BuildmasterConfig = {} @@ -29,9 +29,10 @@ c['caches'] = { ################## -c['slaves'] = [] -for s in MAC_SLAVES + CROSS_SLAVES + LINUX_SLAVES + WINDOWS_SLAVES: - c['slaves'].append(buildslave.BuildSlave(s, SLAVE_PASSWORD, max_builds=1)) +c['slaves'] = [ + buildslave.BuildSlave(s, SLAVE_PASSWORD, max_builds=1) + for s in MAC_WORKERS + CROSS_WORKERS + LINUX_WORKERS + WINDOWS_WORKERS +] ################## @@ -145,18 +146,29 @@ def branch_priority(builder, requests): return requests[0] +def workers_for_env(env): + return { + envs.build_android: CROSS_WORKERS, + envs.build_arm32: CROSS_WORKERS, + envs.build_arm64: CROSS_WORKERS, + envs.build_linux: LINUX_WORKERS, + envs.build_mac: MAC_WORKERS, + envs.build_windows_msvc: WINDOWS_WORKERS, + }[env] + + class DynamicServoBuilder(util.BuilderConfig): """\ Builder which uses DynamicServoFactory to run steps from a YAML file in the main servo repo. """ - def __init__(self, name, slavenames, environment): + def __init__(self, name, environment): # util.BuilderConfig is an old-style class so we cannot use super() # but must hardcode the superclass here util.BuilderConfig.__init__( self, name=name, - slavenames=slavenames, + slavenames=workers_for_env(environment), factory=factories.DynamicServoFactory(name, environment), nextBuild=branch_priority, canStartBuild=util.enforceChosenSlave, @@ -164,34 +176,31 @@ class DynamicServoBuilder(util.BuilderConfig): c['builders'] = [ - DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), - DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), - DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32), - DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64), - DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-nightly", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-rel-css", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-rel-intermittent", LINUX_SLAVES, - envs.build_linux), - DynamicServoBuilder("linux-rel-nogate", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("linux-rel-wpt", LINUX_SLAVES, envs.build_linux), - DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-nightly", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-css1", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-css2", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-intermittent", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt1", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt2", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt3", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("mac-rel-wpt4", MAC_SLAVES, envs.build_mac), - DynamicServoBuilder("windows-msvc-dev", WINDOWS_SLAVES, - envs.build_windows_msvc), - DynamicServoBuilder("windows-msvc-nightly", WINDOWS_SLAVES, - envs.build_windows_msvc), + DynamicServoBuilder("android", envs.build_android), + DynamicServoBuilder("android-nightly", envs.build_android), + DynamicServoBuilder("arm32", envs.build_arm32), + DynamicServoBuilder("arm64", envs.build_arm64), + DynamicServoBuilder("linux-dev", envs.build_linux), + DynamicServoBuilder("linux-nightly", envs.build_linux), + DynamicServoBuilder("linux-rel-css", envs.build_linux), + DynamicServoBuilder("linux-rel-intermittent", envs.build_linux), + DynamicServoBuilder("linux-rel-nogate", envs.build_linux), + DynamicServoBuilder("linux-rel-wpt", envs.build_linux), + DynamicServoBuilder("mac-dev-unit", envs.build_mac), + DynamicServoBuilder("mac-nightly", envs.build_mac), + DynamicServoBuilder("mac-rel-css1", envs.build_mac), + DynamicServoBuilder("mac-rel-css2", envs.build_mac), + DynamicServoBuilder("mac-rel-intermittent", envs.build_mac), + DynamicServoBuilder("mac-rel-wpt1", envs.build_mac), + DynamicServoBuilder("mac-rel-wpt2", envs.build_mac), + DynamicServoBuilder("mac-rel-wpt3", envs.build_mac), + DynamicServoBuilder("mac-rel-wpt4", envs.build_mac), + DynamicServoBuilder("windows-msvc-dev", envs.build_windows_msvc), + DynamicServoBuilder("windows-msvc-nightly", envs.build_windows_msvc), # The below builders are not dynamic but rather have hard-coded factories util.BuilderConfig( name="doc", - slavenames=LINUX_SLAVES, + slavenames=LINUX_WORKERS, factory=factories.doc, canStartBuild=util.enforceChosenSlave, ), From 81e1d58c3b594fb07010368302d5a70a6047e14d Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Thu, 19 Oct 2017 08:02:06 -0700 Subject: [PATCH 2/2] Ignore pyc files --- .gitignore | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c8b58f0e..4ff2854e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -.vagrant/ -__pycache__ +/.vagrant + +*.pyc +__pycache__/