diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 99b2bc585f6a..912ed982ddfe 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -659,6 +659,8 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, .with_script("tar -xzf target.tar.gz") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) .with_max_run_time_minutes(90) + .with_retry_on_exit_codes(TASKCLUSTER_RETRY_EXIT_CODE) + .with_max_auto_retries(1) # This does not include the initial run .with_env( TOTAL_CHUNKS=str(total_chunks), THIS_CHUNK=str(this_chunk), @@ -1005,5 +1007,9 @@ def magicleap_nightly(): CONFIG.windows_worker_type = "win2016" CONFIG.macos_worker_type = "macos" +# Must match the constant with the same name in `python/servo/testing_commands.py` +TASKCLUSTER_RETRY_EXIT_CODE = 17 + + if __name__ == "__main__": # pragma: no cover main(task_for=os.environ["TASK_FOR"]) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index acd8556ad793..5fdc75493225 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -145,6 +145,7 @@ def __init__(self, name): self.extra = {} self.treeherder_required = False self.priority = None # Defaults to 'lowest' + self.max_auto_retries = None # Defaults to 5 # All `with_*` methods return `self`, so multiple method calls can be chained. with_description = chaining(setattr, "description") @@ -155,6 +156,7 @@ def __init__(self, name): with_expires_in = chaining(setattr, "expires_in") with_index_and_artifacts_expire_in = chaining(setattr, "index_and_artifacts_expire_in") with_priority = chaining(setattr, "priority") + with_max_auto_retries = chaining(setattr, "max_auto_retries") with_dependencies = chaining(append_to_attr, "dependencies") with_scopes = chaining(append_to_attr, "scopes") @@ -251,6 +253,7 @@ def create(self): routes=routes, extra=self.extra, priority=self.priority, + retries=self.max_auto_retries, ) task_id = taskcluster.slugId() @@ -315,10 +318,12 @@ def __init__(self, *args, **kwargs): self.features = {} self.mounts = [] self.artifacts = [] + self.retry_on_exit_codes = [] with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") with_mounts = chaining(append_to_attr, "mounts") with_env = chaining(update_attr, "env") + with_retry_on_exit_codes = chaining(append_to_attr, "retry_on_exit_codes") def build_command(self): # pragma: no cover """ @@ -337,6 +342,9 @@ def build_worker_payload(self): "command": self.build_command(), "maxRunTime": self.max_run_time_minutes * 60 } + # https://github.com/taskcluster/generic-worker/blob/a8fe0120f3/simple_posix.yml#L224-L249 + if self.retry_on_exit_codes: + worker_payload["onExitStatus"] = {"retry": self.retry_on_exit_codes} return dict_update_if_truthy( worker_payload, env=self.env, diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 1aa2b0961e50..ea67881d1207 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -43,6 +43,10 @@ from servo_tidy import tidy from servo_tidy_tests import test_tidy + +# Must match the constant with the same name in `etc/taskcluster/decision_task.py` +TASKCLUSTER_RETRY_EXIT_CODE = 17 + SCRIPT_PATH = os.path.split(__file__)[0] PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..")) WEB_PLATFORM_TESTS_PATH = os.path.join("tests", "wpt", "web-platform-tests") @@ -577,7 +581,7 @@ def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, if len(actual_failures) == 0: return 0 - return 1 + return TASKCLUSTER_RETRY_EXIT_CODE @Command('test-android-startup', description='Extremely minimal testing of Servo for Android',