diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index f892afab..97d752a1 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -4,6 +4,7 @@ from buildbot.plugins import steps, util from buildbot.process import buildstep from buildbot.status.results import SUCCESS +from twisted.internet import defer import yaml import environments as envs @@ -104,6 +105,104 @@ def make_step(self, command): return step_class(**step_kwargs) +class StepsYAMLParsingStep(buildstep.ShellMixin, buildstep.BuildStep): + """ + Step which resolves the in-tree YAML and dynamically adds test steps. + """ + + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, builder_name, environment, yaml_path, **kwargs): + kwargs = self.setupShellMixin(kwargs) + buildstep.BuildStep.__init__(self, **kwargs) + self.builder_name = builder_name + self.environment = environment + self.yaml_path = yaml_path + + @defer.inlineCallbacks + def run(self): + try: + print_yaml_cmd = "cat {}".format(self.yaml_path) + cmd = yield self.makeRemoteShellCommand( + command=[print_yaml_cmd], + collectStdout=True + ) + yield self.runCommand(cmd) + + result = cmd.results() + if result != util.SUCCESS: + raise Exception("Command failed with return code: {}" .format( + str(cmd.rc) + )) + else: + builder_steps = yaml.safe_load(cmd.stdout) + commands = builder_steps[self.builder_name] + dynamic_steps = [self.make_step(command) + for command in commands] + except Exception as e: # Bad step configuration, fail build + # Capture the exception and re-raise with a friendly message + raise Exception("Bad step configuration for {}: {}".format( + self.builder_name, + str(e) + )) + + # TODO: windows compatibility (use a custom script for this?) + pkill_step = steps.ShellCommand(command=["pkill", "-x", "servo"], + decodeRC={0: SUCCESS, 1: SUCCESS}) + static_steps = [pkill_step] + + self.build.steps += static_steps + dynamic_steps + + defer.returnValue(result) + + def make_step(self, command): + step_kwargs = {} + step_kwargs['env'] = self.environment + + command = command.split(' ') + step_kwargs['command'] = command + + step_class = steps.ShellCommand + args = iter(command) + for arg in args: + # Change Step class to capture warnings as needed + # (steps.Compile and steps.Test catch warnings) + if arg == './mach': + mach_arg = next(args) + if re.match('build(-.*)?', mach_arg): + step_class = steps.Compile + elif re.match('test-.*', mach_arg): + step_class = steps.Test + + # Capture any logfiles + elif re.match('--log-.*', arg): + logfile = next(args) + if 'logfiles' not in step_kwargs: + step_kwargs['logfiles'] = {} + step_kwargs['logfiles'][logfile] = logfile + + return step_class(**step_kwargs) + + +class DynamicServoYAMLFactory(ServoFactory): + """ + Smart factory which takes a list of shell commands from a YAML file + located in the main servo/servo repository and creates the appropriate + Buildbot Steps. + Uses heuristics to infer Step type, if there are any logfiles, etc. + """ + + def __init__(self, builder_name, environment): + + # util.BuildFactory is an old-style class so we cannot use super() + # but must hardcode the superclass here + ServoFactory.__init__(self, [ + StepsYAMLParsingStep(builder_name, environment, + "etc/ci/buildbot_steps.yml") + ]) + + doc = ServoFactory([ # This is not dynamic because a) we need to pass the logEnviron kwarg # and b) changes to the documentation build are already encapsulated diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index f4b14ce9..da2b0436 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -65,6 +65,7 @@ c['schedulers'].append(schedulers.AnyBranchScheduler( builderNames=[ "linux-dev", "linux-rel", + "linux-dev-yaml", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", @@ -86,6 +87,7 @@ c['schedulers'].append(schedulers.ForceScheduler( builderNames=[ "linux-dev", "linux-rel", + "linux-dev-yaml", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", @@ -134,6 +136,23 @@ class DynamicServoBuilder(util.BuilderConfig): ) +class DynamicServoYAMLBuilder(util.BuilderConfig): + """ + Builder which uses DynamicServoYAMLFactory to run steps from a + yaml file in the Servo source tree. + """ + def __init__(self, name, slavenames, environment): + # util.BuilderConfig is an old-style class so we cannot use super() + # but must hardcode the superclass here + util.BuilderConfig.__init__( + self, + name=name, + slavenames=slavenames, + factory=factories.DynamicServoYAMLFactory(name, environment), + nextBuild=branch_priority, + category="auto", + ) + c['builders'] = [ DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux), DynamicServoBuilder("linux-rel", LINUX_SLAVES, envs.build_linux), @@ -158,6 +177,9 @@ c['builders'] = [ factory=factories.doc, category="auto", ), + # Testing the In Tree YAML build + DynamicServoYAMLBuilder("linux-dev-yaml", LINUX_SLAVES, + envs.build_linux) ]