From 7f65a6525bbf8db23fb4595c69732bab543b67b4 Mon Sep 17 00:00:00 2001 From: Ostap Zherebetskyi Date: Mon, 16 Nov 2026 12:54:04 +0200 Subject: [PATCH 1/6] Add Celery stress test suite management and metrics tracking --- admin/base/celery_panel.py | 69 +- admin/base/forms.py | 1 + admin/base/urls.py | 5 +- .../celery_stress_testing.html | 738 +++++++++--------- .../dj_celery_panel/stress_suite_detail.html | 179 +++++ .../dj_celery_panel/stress_suite_table.html | 17 + .../admin/dj_celery_panel/stress_suites.html | 21 + osf/celery_stress_test/control.py | 85 ++ osf/celery_stress_test/metrics.py | 55 ++ osf/celery_stress_test/tasks.py | 238 ++++-- osf/migrations/0061_celery_test_control.py | 61 ++ osf/models/__init__.py | 1 + osf/models/celery_test.py | 71 ++ osf_tests/test_celery_stress_control.py | 381 +++++++++ 14 files changed, 1481 insertions(+), 441 deletions(-) create mode 100644 admin/templates/admin/dj_celery_panel/stress_suite_detail.html create mode 100644 admin/templates/admin/dj_celery_panel/stress_suite_table.html create mode 100644 admin/templates/admin/dj_celery_panel/stress_suites.html create mode 100644 osf/celery_stress_test/control.py create mode 100644 osf/celery_stress_test/metrics.py create mode 100644 osf/migrations/0061_celery_test_control.py create mode 100644 osf/models/celery_test.py create mode 100644 osf_tests/test_celery_stress_control.py diff --git a/admin/base/celery_panel.py b/admin/base/celery_panel.py index 5a9c2927ce8..cdf940c21b1 100644 --- a/admin/base/celery_panel.py +++ b/admin/base/celery_panel.py @@ -3,16 +3,22 @@ from functools import cache from django.contrib import messages -from django.shortcuts import redirect, render +from django.core.paginator import Paginator +from django.db import transaction +from django.db.models import Count, Q +from django.forms import formset_factory +from django.shortcuts import get_object_or_404, redirect, render from django.views.decorators.http import require_http_methods from dj_celery_panel import views from dj_celery_panel.celery_utils import CeleryQueuesInspectBackend, QueueDetailPage, QueueListPage from dj_celery_panel.conf import panel_config from framework.celery_tasks import app as celery_app +from osf.models.celery_test import CeleryTestSuite, CeleryTestTask +from osf.celery_stress_test.control import cancel_suite, finish_publishing +from osf.celery_stress_test.metrics import suite_metrics from osf.celery_stress_test.tasks import start_celery_stress_test from .forms import CeleryStressTestForm, WorkloadForm -from django.forms import formset_factory WorkloadFormSet = formset_factory( @@ -136,6 +142,7 @@ def get_unmatched_routing_entries_by_queue(self): return unmatched_entries +@transaction.non_atomic_requests @panel_config.permission_required('celery_stress_test') @require_http_methods(['GET', 'POST']) def celery_stress_test_view(request): @@ -168,9 +175,15 @@ def celery_stress_test_view(request): 'Please configure at least one workload.', ) else: + suite = CeleryTestSuite.objects.create( + name=config['name'], config=config, created_by=request.user, + ) try: - result = start_celery_stress_test.delay(config) - except Exception: + start_celery_stress_test.apply_async( + args=[str(suite.id)], task_id=str(suite.coordinator_task_id), retry=False, + ) + except Exception as error: + finish_publishing(suite.id, f'Coordinator publication failed: {error}') messages.error( request, 'Failed to start the stress test. Please try again.', @@ -178,9 +191,9 @@ def celery_stress_test_view(request): else: messages.success( request, - f"Stress test submitted successfully. Task ID: {result.id}", + f"Stress test submitted successfully: {suite.name}", ) - return redirect(request.path) + return redirect('celery_stress_suite_detail', suite_id=suite.id) else: form = CeleryStressTestForm() workload_formset = WorkloadFormSet() @@ -195,3 +208,47 @@ def celery_stress_test_view(request): 'admin/dj_celery_panel/celery_stress_testing.html', context, ) + + +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['GET']) +def celery_stress_suites(request): + context = panel_config.get_context(request, current_tab='celery_stress_test') + suites = CeleryTestSuite.objects.annotate( + task_count=Count('tasks'), + succeeded_count=Count('tasks', filter=Q(tasks__status=CeleryTestTask.Status.SUCCEEDED)), + failed_count=Count('tasks', filter=Q(tasks__status=CeleryTestTask.Status.FAILED)), + ) + context.update({ + 'active_suites': suites.filter(status__in=CeleryTestSuite.ACTIVE_STATUSES), + 'history': Paginator(suites.exclude(status__in=CeleryTestSuite.ACTIVE_STATUSES), 25).get_page(request.GET.get('page')), + }) + return render(request, 'admin/dj_celery_panel/stress_suites.html', context) + + +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['GET']) +def celery_stress_suite_detail(request, suite_id): + suite = get_object_or_404(CeleryTestSuite, pk=suite_id) + context = panel_config.get_context(request, current_tab='celery_stress_test') + context.update({ + 'suite': suite, + 'metrics': suite_metrics(suite), + 'task_page': Paginator(suite.tasks.order_by('-submitted_at', '-id'), 50).get_page(request.GET.get('page')), + }) + return render(request, 'admin/dj_celery_panel/stress_suite_detail.html', context) + + +@transaction.non_atomic_requests +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['POST']) +def celery_stress_suite_cancel(request, suite_id): + get_object_or_404(CeleryTestSuite, pk=suite_id) + suite, warning = cancel_suite(suite_id, request.user, celery_app) + if warning: + messages.warning(request, warning) + elif suite.cancel_requested_at: + messages.success(request, 'Cancellation requested. Queued tasks are revoked; running tasks and the coordinator stop cooperatively.') + else: + messages.info(request, 'This suite has already finished.') + return redirect('celery_stress_suite_detail', suite_id=suite.id) diff --git a/admin/base/forms.py b/admin/base/forms.py index 11c9386693f..ea43f5038e4 100644 --- a/admin/base/forms.py +++ b/admin/base/forms.py @@ -27,6 +27,7 @@ class ArchiveRegistrationWithPigeonForm(forms.Form): class CeleryStressTestForm(forms.Form): + name = forms.CharField(max_length=100, initial='Celery load test') total_duration = forms.IntegerField( label='Duration (seconds)', min_value=1, diff --git a/admin/base/urls.py b/admin/base/urls.py index 533fedabc9b..6484e3f2914 100644 --- a/admin/base/urls.py +++ b/admin/base/urls.py @@ -15,7 +15,10 @@ re_path(r'^unprotected_views/$', views.unprotected_views, name='unprotected_views'), re_path(r'^middleware_check/$', views.admin_auth_middleware_check, name='middleware_check'), re_path(r'^admin/dj-celery-panel/$', celery_panel.overview), - re_path(r'^admin/dj-celery-panel/celery_stress_test/$', celery_panel.celery_stress_test_view, name='celery_stress_test'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/$', celery_panel.celery_stress_suites, name='celery_stress_test'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/new/$', celery_panel.celery_stress_test_view, name='celery_stress_test_new'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/$', celery_panel.celery_stress_suite_detail, name='celery_stress_suite_detail'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/cancel/$', celery_panel.celery_stress_suite_cancel, name='celery_stress_suite_cancel'), re_path(r'^admin/dj-celery-panel/', include('dj_celery_panel.urls')), re_path(r'^admin/', admin.site.urls), re_path(r'^provider_asset_files/', include('admin.provider_asset_files.urls', namespace='provider_asset_files')), diff --git a/admin/templates/admin/dj_celery_panel/celery_stress_testing.html b/admin/templates/admin/dj_celery_panel/celery_stress_testing.html index 1d600ded786..0eeeb01174f 100644 --- a/admin/templates/admin/dj_celery_panel/celery_stress_testing.html +++ b/admin/templates/admin/dj_celery_panel/celery_stress_testing.html @@ -2,122 +2,222 @@ {% load i18n static dcr_icons %} {% block panel_page_content %} -
-
-
- {% dcr_icon "cog" %} -
-
-

{% trans "Stress Testing" %}

-

- {% trans "Run stress tests on your Celery workers to evaluate their performance and stability under load." %} -

+
+
+
+ {% dcr_icon "cog" %} +
+
+

{% trans "Stress Testing" %}

+

+ {% trans "Run stress tests on your Celery workers to evaluate their performance and stability under load." %} +

+
-
-
- {% csrf_token %} +

{% trans "All test suites" %}

+ + {% csrf_token %} - {% if form.non_field_errors %} -
- {{ form.non_field_errors }} -
- {% endif %} - - {# Global configuration #} -
-
-

{% trans "Global Configuration" %}

-

- {% trans "Configure the duration and maximum publishing rate for the entire test suite." %} -

-
+ {% if form.non_field_errors %} +
+ {{ form.non_field_errors }} +
+ {% endif %} -
-
+ {# Global configuration #} +
+
+

{% trans "Global Configuration" %}

+

+ {% trans "Configure the duration and maximum publishing rate for the entire test suite." %} +

+
+ +
- - {{ form.total_duration }} - {% if form.total_duration.help_text %} - {{ form.total_duration.help_text }} - {% endif %} - {{ form.total_duration.errors }} + {{ form.name.label_tag }} {{ form.name }} {{ form.name.errors }}
+
+
+ + {{ form.total_duration }} + {% if form.total_duration.help_text %} + {{ form.total_duration.help_text }} + {% endif %} + {{ form.total_duration.errors }} +
-
- - {{ form.max_total_rate }} - {% if form.max_total_rate.help_text %} - {{ form.max_total_rate.help_text }} - {% endif %} - {{ form.max_total_rate.errors }} +
+ + {{ form.max_total_rate }} + {% if form.max_total_rate.help_text %} + {{ form.max_total_rate.help_text }} + {% endif %} + {{ form.max_total_rate.errors }} +
-
-
- - {# Workload configuration #} -
-
-
-

{% trans "Workloads" %}

-

- {% trans "Define the workloads to publish to your Celery queues." %} -

-
+
- -
- {{ workload_formset.max_num }} + {# Workload configuration #} +
+
+
+

{% trans "Workloads" %}

+

+ {% trans "Define the workloads to publish to your Celery queues." %} +

+
+ + +
+ {{ workload_formset.max_num }} +
-
- {{ workload_formset.management_form }} + {{ workload_formset.management_form }} - {% if workload_formset.non_form_errors %} -
- {{ workload_formset.non_form_errors }} -
- {% endif %} + {% if workload_formset.non_form_errors %} +
+ {{ workload_formset.non_form_errors }} +
+ {% endif %} + +
+ {% for workload_form in workload_formset %} +
+
+
+

+ {% blocktrans with number=forloop.counter %} + Workload {{ number }} + {% endblocktrans %} +

+

+ {% trans "Configure the workload type, destination queue, and publishing rate." %} +

+
-
- {% for workload_form in workload_formset %} -
-
-
-

- {% blocktrans with number=forloop.counter %} - Workload {{ number }} - {% endblocktrans %} -

-

- {% trans "Configure the workload type, destination queue, and publishing rate." %} -

+
- -
+
+ {{ workload_form.id }} -
- {{ workload_form.id }} + {% if workload_form.non_field_errors %} +
+ {{ workload_form.non_field_errors }} +
+ {% endif %} + +
+
+ + {{ workload_form.name }} + {{ workload_form.name.errors }} +
+ +
+ + {{ workload_form.mode }} + {{ workload_form.mode.errors }} +
- {% if workload_form.non_field_errors %} -
- {{ workload_form.non_field_errors }} +
+ + {{ workload_form.queue }} + {{ workload_form.queue.errors }} +
+ +
+ + {{ workload_form.rate }} + {{ workload_form.rate.errors }} +
- {% endif %} + +
+

{% trans "Workload Arguments" %}

+ +
+
+ + {{ workload_form.io_seconds }} + {{ workload_form.io_seconds.errors }} + + {% trans "Time spent waiting in I/O and mixed modes." %} + +
+
+ +
+
+ + {{ workload_form.cpu_iterations }} + {{ workload_form.cpu_iterations.errors }} + + {% trans "Number of calculation iterations in CPU and mixed modes." %} + +
+
+
+
+
+ {% empty %} +

+ {% trans "No workloads configured. Add a workload to get started." %} +

+ {% endfor %} +
+
+ + {# Hidden template used by JavaScript to add new workloads #} + + +
+
- +
- {# Hidden template used by JavaScript to add new workloads #} -