diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index c966d9f0..5eb056ff 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -1,4 +1,5 @@ import copy +import os.path import re from buildbot.plugins import steps, util @@ -14,12 +15,12 @@ class ServoFactory(util.BuildFactory): - """\ + """ Build factory which checks out the servo repo as the first build step. """ def __init__(self, build_steps): - """\ + """ Takes a list of Buildbot steps. Prefer using DynamicServoFactory to using this class directly. """ @@ -32,10 +33,138 @@ def __init__(self, build_steps): util.BuildFactory.__init__(self, all_steps) +class BadConfigurationStep(buildstep.BuildStep): + """ + Step which immediately fails the build due to a bad configuration. + """ + + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, exception): + self.exception = exception + buildstep.BuildStep.__init__(self) + + def run(self): + self.addCompleteLog("bad_configuration.txt", str(self.exception)) + raise Exception("Bad configuration, unable to convert to steps" + + str(self.exception)) + + +class DynamicServoFactory(ServoFactory): + """ + Smart factory which takes a list of shell commands from a yaml file + and creates the appropriate Buildbot Steps. Uses heuristics to infer + Step type, if there are any logfiles, etc. + """ + + def __init__(self, builder_name, environment): + self.environment = environment + self.is_windows = re.match('windows.*', builder_name) is not None + self.builder_name = builder_name + try: + config_dir = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(config_dir, 'steps.yml') + with open(yaml_path) as steps_file: + builder_steps = yaml.safe_load(steps_file) + commands = builder_steps[builder_name] + dynamic_steps = [self.make_step(command) for command in commands] + except Exception as e: # Bad step configuration, fail the build + dynamic_steps = [BadConfigurationStep(e)] + + pkill_step = [self.make_pkill_step("servo")] + + # util.BuildFactory is an old-style class so we cannot use super() + # but must hardcode the superclass here + ServoFactory.__init__(self, pkill_step + dynamic_steps) + + def make_step(self, command): + step_kwargs = {} + step_env = copy.deepcopy(self.environment) + + command = command.split(' ') + + # Add `bash -l` before every command on Windows builders + bash_args = ["bash", "-l"] if self.is_windows else [] + step_kwargs['command'] = bash_args + command + if self.is_windows: + step_env += envs.Environment({ + # Set home directory, to avoid adding `cd` command every time + 'HOME': r'C:\buildbot\slave\{}\build'.format( + self.builder_name + ), + }) + + step_desc = [] + step_class = steps.ShellCommand + args = iter(command) + for arg in args: + # Change Step class to capture warnings as needed + # (steps.Compile and steps.Test catch warnings) + if arg == './mach': + mach_arg = next(args) + step_desc = [mach_arg] + if re.match('build(-.*)?', mach_arg): + step_class = steps.Compile + elif re.match('package', mach_arg): + step_class = steps.Compile + elif re.match('test-.*', mach_arg): + step_class = steps.Test + + # Capture any logfiles + elif re.match('--log-.*', arg): + logfile = next(args) + if 'logfiles' not in step_kwargs: + step_kwargs['logfiles'] = {} + step_kwargs['logfiles'][logfile] = logfile + + # Provide environment variables for s3cmd + elif arg == './etc/ci/upload_nightly.sh': + step_kwargs['logEnviron'] = False + step_env += envs.upload_nightly + if self.is_windows: + # s3cmd on Windows only works within msys + step_env['MSYSTEM'] = 'MSYS' + step_env['PATH'] = ';'.join([ + r'C:\msys64\usr\bin', + r'C:\Windows\system32', + r'C:\Windows', + r'C:\Windows\System32\Wbem', + r'C:\Windows\System32\WindowsPowerShell\v1.0', + r'C:\Program Files\Amazon\cfn-bootstrap', + ]) + + # Set token for homebrew repository + elif arg == './etc/ci/update_brew.sh': + step_kwargs['logEnviron'] = False + step_env += envs.update_brew + + else: + step_desc += [arg] + + if step_class != steps.ShellCommand: + step_kwargs['description'] = "running" + step_kwargs['descriptionDone'] = "ran" + step_kwargs['descriptionSuffix'] = " ".join(step_desc) + + step_kwargs['env'] = step_env + return step_class(**step_kwargs) + + def make_pkill_step(self, target): + if self.is_windows: + pkill_command = ["powershell", "kill", "-n", target] + else: + pkill_command = ["pkill", "-x", target] + + return steps.ShellCommand( + command=pkill_command, + decodeRC={0: SUCCESS, 1: SUCCESS} + ) + + class StepsYAMLParsingStep(buildstep.ShellMixin, buildstep.BuildStep): - """\ - Step which reads the YAML steps configuration in the main servo repo - and dynamically adds test steps. + """ + Step which resolves the in-tree YAML and dynamically adds test steps. """ haltOnFailure = True @@ -181,11 +310,11 @@ def make_pkill_step(self, target): ) -class DynamicServoFactory(ServoFactory): - """\ - Smart factory which takes a list of shell commands - from a YAML file located in the main servo/servo repository - and creates the appropriate Buildbot Steps. +class DynamicServoYAMLFactory(ServoFactory): + """ + Smart factory which takes a list of shell commands from a YAML file + located in the main servo/servo repository and creates the appropriate + Buildbot Steps. Uses heuristics to infer Step type, if there are any logfiles, etc. """ diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 2b55a608..8ceeb086 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -67,6 +67,7 @@ c['schedulers'].append(schedulers.AnyBranchScheduler( "arm32", "arm64", "linux-dev", + "linux-dev-yaml", "linux-rel-css", "linux-rel-wpt", "mac-dev-unit", @@ -91,6 +92,7 @@ c['schedulers'].append(schedulers.ForceScheduler( "arm32", "arm64", "linux-dev", + "linux-dev-yaml", "linux-nightly", "linux-rel-css", "linux-rel-wpt", @@ -134,9 +136,8 @@ def branch_priority(builder, requests): class DynamicServoBuilder(util.BuilderConfig): - """\ - Builder which uses DynamicServoFactory to run steps - from a YAML file in the main servo repo. + """ + Builder which uses DynamicServoFactory to run steps from a yaml file. """ def __init__(self, name, slavenames, environment): # util.BuilderConfig is an old-style class so we cannot use super() @@ -150,6 +151,22 @@ class DynamicServoBuilder(util.BuilderConfig): ) +class DynamicServoYAMLBuilder(util.BuilderConfig): + """ + Builder which uses DynamicServoYAMLFactory to run steps from a + yaml file in the Servo source tree. + """ + def __init__(self, name, slavenames, environment): + # util.BuilderConfig is an old-style class so we cannot use super() + # but must hardcode the superclass here + util.BuilderConfig.__init__( + self, + name=name, + slavenames=slavenames, + factory=factories.DynamicServoYAMLFactory(name, environment), + nextBuild=branch_priority, + ) + c['builders'] = [ DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), @@ -177,6 +194,9 @@ c['builders'] = [ slavenames=LINUX_SLAVES, factory=factories.doc, ), + # Testing the In Tree YAML build + DynamicServoYAMLBuilder("linux-dev-yaml", LINUX_SLAVES, + envs.build_linux) ]