From 4075ac4b78eae67d89f4d6c9b4206354162a44a0 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Thu, 6 Oct 2016 01:39:51 -0400 Subject: [PATCH] Remove old DynamicServoFactory which reads from saltfs Now that the new dynamic factory is working (which reads the steps configuration from the main servo repo), remove the old one. However, re-use the old name (DynamicServoFactory instead of DynamicServoYAMLFactory) for concision and clarity. --- buildbot/master/files/config/factories.py | 149 ++-------------------- buildbot/master/files/config/master.cfg | 26 +--- 2 files changed, 13 insertions(+), 162 deletions(-) diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index 278b0ea0..314ed64b 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -1,5 +1,4 @@ import copy -import os.path import re from buildbot.plugins import steps, util @@ -15,12 +14,12 @@ class ServoFactory(util.BuildFactory): - """ + """\ Build factory which checks out the servo repo as the first build step. """ def __init__(self, build_steps): - """ + """\ Takes a list of Buildbot steps. Prefer using DynamicServoFactory to using this class directly. """ @@ -33,138 +32,10 @@ def __init__(self, build_steps): util.BuildFactory.__init__(self, all_steps) -class BadConfigurationStep(buildstep.BuildStep): - """ - Step which immediately fails the build due to a bad configuration. - """ - - haltOnFailure = True - flunkOnFailure = True - - def __init__(self, exception): - self.exception = exception - buildstep.BuildStep.__init__(self) - - def run(self): - self.addCompleteLog("bad_configuration.txt", str(self.exception)) - raise Exception("Bad configuration, unable to convert to steps" + - str(self.exception)) - - -class DynamicServoFactory(ServoFactory): - """ - Smart factory which takes a list of shell commands from a yaml file - and creates the appropriate Buildbot Steps. Uses heuristics to infer - Step type, if there are any logfiles, etc. - """ - - def __init__(self, builder_name, environment): - self.environment = environment - self.is_windows = re.match('windows.*', builder_name) is not None - self.builder_name = builder_name - try: - config_dir = os.path.dirname(os.path.realpath(__file__)) - yaml_path = os.path.join(config_dir, 'steps.yml') - with open(yaml_path) as steps_file: - builder_steps = yaml.safe_load(steps_file) - commands = builder_steps[builder_name] - dynamic_steps = [self.make_step(command) for command in commands] - except Exception as e: # Bad step configuration, fail the build - dynamic_steps = [BadConfigurationStep(e)] - - pkill_step = [self.make_pkill_step("servo")] - - # util.BuildFactory is an old-style class so we cannot use super() - # but must hardcode the superclass here - ServoFactory.__init__(self, pkill_step + dynamic_steps) - - def make_step(self, command): - step_kwargs = {} - step_env = copy.deepcopy(self.environment) - - command = command.split(' ') - - # Add `bash -l` before every command on Windows builders - bash_args = ["bash", "-l"] if self.is_windows else [] - step_kwargs['command'] = bash_args + command - if self.is_windows: - step_env += envs.Environment({ - # Set home directory, to avoid adding `cd` command every time - 'HOME': r'C:\buildbot\slave\{}\build'.format( - self.builder_name - ), - }) - - step_desc = [] - step_class = steps.ShellCommand - args = iter(command) - for arg in args: - # Change Step class to capture warnings as needed - # (steps.Compile and steps.Test catch warnings) - if arg == './mach': - mach_arg = next(args) - step_desc = [mach_arg] - if re.match('build(-.*)?', mach_arg): - step_class = steps.Compile - elif re.match('package', mach_arg): - step_class = steps.Compile - elif re.match('test-.*', mach_arg): - step_class = steps.Test - - # Capture any logfiles - elif re.match('--log-.*', arg): - logfile = next(args) - if 'logfiles' not in step_kwargs: - step_kwargs['logfiles'] = {} - step_kwargs['logfiles'][logfile] = logfile - - # Provide environment variables for s3cmd - elif arg == './etc/ci/upload_nightly.sh': - step_kwargs['logEnviron'] = False - step_env += envs.upload_nightly - if self.is_windows: - # s3cmd on Windows only works within msys - step_env['MSYSTEM'] = 'MSYS' - step_env['PATH'] = ';'.join([ - r'C:\msys64\usr\bin', - r'C:\Windows\system32', - r'C:\Windows', - r'C:\Windows\System32\Wbem', - r'C:\Windows\System32\WindowsPowerShell\v1.0', - r'C:\Program Files\Amazon\cfn-bootstrap', - ]) - - # Set token for homebrew repository - elif arg == './etc/ci/update_brew.sh': - step_kwargs['logEnviron'] = False - step_env += envs.update_brew - - else: - step_desc += [arg] - - if step_class != steps.ShellCommand: - step_kwargs['description'] = "running" - step_kwargs['descriptionDone'] = "ran" - step_kwargs['descriptionSuffix'] = " ".join(step_desc) - - step_kwargs['env'] = step_env - return step_class(**step_kwargs) - - def make_pkill_step(self, target): - if self.is_windows: - pkill_command = ["powershell", "kill", "-n", target] - else: - pkill_command = ["pkill", "-x", target] - - return steps.ShellCommand( - command=pkill_command, - decodeRC={0: SUCCESS, 1: SUCCESS} - ) - - class StepsYAMLParsingStep(buildstep.ShellMixin, buildstep.BuildStep): - """ - Step which resolves the in-tree YAML and dynamically adds test steps. + """\ + Step which reads the YAML steps configuration in the main servo repo + and dynamically adds test steps. """ haltOnFailure = True @@ -309,11 +180,11 @@ def make_pkill_step(self, target): ) -class DynamicServoYAMLFactory(ServoFactory): - """ - Smart factory which takes a list of shell commands from a YAML file - located in the main servo/servo repository and creates the appropriate - Buildbot Steps. +class DynamicServoFactory(ServoFactory): + """\ + Smart factory which takes a list of shell commands + from a YAML file located in the main servo/servo repository + and creates the appropriate Buildbot Steps. Uses heuristics to infer Step type, if there are any logfiles, etc. """ diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 8ceeb086..2b55a608 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -67,7 +67,6 @@ c['schedulers'].append(schedulers.AnyBranchScheduler( "arm32", "arm64", "linux-dev", - "linux-dev-yaml", "linux-rel-css", "linux-rel-wpt", "mac-dev-unit", @@ -92,7 +91,6 @@ c['schedulers'].append(schedulers.ForceScheduler( "arm32", "arm64", "linux-dev", - "linux-dev-yaml", "linux-nightly", "linux-rel-css", "linux-rel-wpt", @@ -136,8 +134,9 @@ def branch_priority(builder, requests): class DynamicServoBuilder(util.BuilderConfig): - """ - Builder which uses DynamicServoFactory to run steps from a yaml file. + """\ + Builder which uses DynamicServoFactory to run steps + from a YAML file in the main servo repo. """ def __init__(self, name, slavenames, environment): # util.BuilderConfig is an old-style class so we cannot use super() @@ -151,22 +150,6 @@ class DynamicServoBuilder(util.BuilderConfig): ) -class DynamicServoYAMLBuilder(util.BuilderConfig): - """ - Builder which uses DynamicServoYAMLFactory to run steps from a - yaml file in the Servo source tree. - """ - def __init__(self, name, slavenames, environment): - # util.BuilderConfig is an old-style class so we cannot use super() - # but must hardcode the superclass here - util.BuilderConfig.__init__( - self, - name=name, - slavenames=slavenames, - factory=factories.DynamicServoYAMLFactory(name, environment), - nextBuild=branch_priority, - ) - c['builders'] = [ DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), @@ -194,9 +177,6 @@ c['builders'] = [ slavenames=LINUX_SLAVES, factory=factories.doc, ), - # Testing the In Tree YAML build - DynamicServoYAMLBuilder("linux-dev-yaml", LINUX_SLAVES, - envs.build_linux) ]