From 8b88d2d88745f73adcc10ea96fb4dfad1ad36dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 30 Aug 2026 23:35:54 +0200 Subject: [PATCH 1/5] [ADD] metrics gathering for Prometheus --- queue_job/jobrunner/channels.py | 83 ++++++++++++++++++++++++++++++++- queue_job/jobrunner/metrics.py | 73 +++++++++++++++++++++++++++++ queue_job/jobrunner/runner.py | 45 +++++++++++++++++- 3 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 queue_job/jobrunner/metrics.py diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index 2e8c6109da..e263bf6004 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -2,6 +2,8 @@ # Copyright 2015-2016 Camptocamp SA # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) import logging +import math +import weakref from collections import namedtuple from dataclasses import asdict, dataclass from functools import total_ordering @@ -10,6 +12,7 @@ from ..exception import ChannelNotFound from ..job import CANCELLED, DONE, ENQUEUED, FAILED, PENDING, STARTED, WAIT_DEPENDENCIES +from . import metrics RELOAD_PAYLOAD = "reload" NOT_DONE = (WAIT_DEPENDENCIES, PENDING, ENQUEUED, STARTED, FAILED) @@ -437,10 +440,12 @@ def __init__( self.parent = parent if self.parent: self.parent.children[name] = self + self.parent._register_channel_gauges() self.children = {} self._queue = ChannelQueue() self._running = set() self._failed = set() + self._waiting_dependencies = set() self._pause_until = 0 # utc seconds since the epoch self.capacity = ( capacity if (capacity is not None) else (parent and parent.capacity_default) @@ -450,6 +455,60 @@ def __init__( self.sequential = sequential or (parent and parent.sequential_default) self.sequential_default = sequential_default self.paused = paused + self._metrics_labels = None + self._register_channel_gauges() + + def __del__(self): + self._unregister_channel_gauges() + + def _register_channel_gauges(self) -> None: + self._unregister_channel_gauges() + self._metrics_labels = { + "channel": self.fullname, + "root": not bool(self.parent), + "leaf": not bool(self.children), + } + metrics.channel_capacity.labels(**self._metrics_labels).set_function( + weakref.proxy(self)._capacity_gauge + ) + metrics.channel_pending.labels(**self._metrics_labels).set_function( + weakref.proxy(self)._pending_gauge + ) + metrics.channel_running.labels(**self._metrics_labels).set_function( + weakref.proxy(self)._running_gauge + ) + metrics.channel_failed.labels(**self._metrics_labels).set_function( + weakref.proxy(self)._failed_gauge + ) + metrics.channel_waiting_dependencies.labels( + **self._metrics_labels + ).set_function(weakref.proxy(self)._waiting_dependencies_gauge) + + def _unregister_channel_gauges(self) -> None: + if not self._metrics_labels: + return + metrics.channel_capacity.remove_by_labels(self._metrics_labels) + metrics.channel_pending.remove_by_labels(self._metrics_labels) + metrics.channel_running.remove_by_labels(self._metrics_labels) + metrics.channel_failed.remove_by_labels(self._metrics_labels) + metrics.channel_waiting_dependencies.remove_by_labels(self._metrics_labels) + + def _capacity_gauge(self) -> float: + if self.capacity is None: + return math.inf + return self.capacity + + def _pending_gauge(self) -> float: + return len(self._queue) + + def _running_gauge(self) -> float: + return len(self._running) + + def _failed_gauge(self) -> float: + return len(self._failed) + + def _waiting_dependencies_gauge(self) -> float: + return len(self._waiting_dependencies) @property def sequential(self): @@ -494,13 +553,14 @@ def get_subchannel_by_name(self, subchannel_name): def __str__(self): capacity = "∞" if self.capacity is None else str(self.capacity) - return "%s(C:%s,Q:%d,R:%d,F:%d%s)" % ( + return "%s(C:%s,Q:%d,R:%d,F:%d%s),W:%s)" % ( self.fullname, capacity, len(self._queue), len(self._running), len(self._failed), ",paused" if self.paused else "", + len(self._waiting_dependencies), ) def remove(self, job): @@ -508,6 +568,7 @@ def remove(self, job): self._queue.remove(job) self._running.discard(job) self._failed.discard(job) + self._waiting_dependencies.discard(job) if self.parent: self.parent.remove(job) @@ -529,6 +590,7 @@ def set_pending(self, job): self._queue.add(job) self._running.discard(job) self._failed.discard(job) + self._waiting_dependencies.discard(job) if self.parent: self.parent.remove(job) _logger.debug("job %s marked pending in channel %s", job.uuid, self) @@ -542,6 +604,7 @@ def set_running(self, job): self._queue.remove(job) self._running.add(job) self._failed.discard(job) + self._waiting_dependencies.discard(job) if self.parent: self.parent.set_running(job) _logger.debug("job %s marked running in channel %s", job.uuid, self) @@ -552,10 +615,23 @@ def set_failed(self, job): self._queue.remove(job) self._running.discard(job) self._failed.add(job) + self._waiting_dependencies.discard(job) if self.parent: self.parent.remove(job) _logger.debug("job %s marked failed in channel %s", job.uuid, self) + def set_waiting_dependencies(self, job): + if job not in self._waiting_dependencies: + self._queue.remove(job) + self._running.discard(job) + self._failed.discard(job) + self._waiting_dependencies.add(job) + if self.parent: + self.parent.remove(job) + _logger.debug( + "job %s marked waiting dependencies in channel %s", job.uuid, self + ) + def has_capacity(self): if self.paused: return False @@ -1179,7 +1255,7 @@ def notify( job.channel.set_failed(job) elif state == WAIT_DEPENDENCIES: # wait until all parent jobs are done - pass + job.channel.set_waiting_dependencies(job) else: _logger.error("unexpected state %s for job %s", state, job) @@ -1205,3 +1281,6 @@ def get_wakeup_time(self): def running_count(self) -> int: """Number of jobs currently running""" return len(self._root_channel._running) + + def _jobs_to_do_gauge(self) -> float: + return len(self._jobs_by_uuid) diff --git a/queue_job/jobrunner/metrics.py b/queue_job/jobrunner/metrics.py new file mode 100644 index 0000000000..f52020699f --- /dev/null +++ b/queue_job/jobrunner/metrics.py @@ -0,0 +1,73 @@ +import socket +from http.server import HTTPServer + +from prometheus_client import CollectorRegistry, Counter, Gauge, MetricsHandler + +_registry = CollectorRegistry() + +_channel_label_names = ("channel", "root", "leaf") +channel_capacity = Gauge( + "queue_job_channel_capacity", + documentation="Channel Capacity", + labelnames=_channel_label_names, + registry=_registry, +) +channel_pending = Gauge( + "queue_job_channel_pending", + documentation="Pending jobs in channel", + labelnames=_channel_label_names, + registry=_registry, +) +channel_running = Gauge( + "queue_job_channel_running", + documentation="Running jobs in channel", + labelnames=_channel_label_names, + registry=_registry, +) +channel_failed = Gauge( + "queue_job_channel_failed", + documentation="Failed jobs in channel", + labelnames=_channel_label_names, + registry=_registry, +) +channel_waiting_dependencies = Gauge( + "queue_job_channel_waiting_dependencies", + documentation="Jobs waiting for dependencies in channel", + labelnames=_channel_label_names, + registry=_registry, +) + +jobs_to_do = Gauge( + "queue_job_jobs_to_do", + documentation=( + "Number of jobs waiting to be done (including running and failed jobs)" + ), + registry=_registry, +) +jobs_scheduled_total = Counter( + "queue_job_jobs_scheduled_total", + documentation=( + "Total number of jobs scheduled for execution (asked Odoo to run job)" + ), + labelnames=("db",), + registry=_registry, +) +dead_jobs_requeued_total = Counter( + "queue_job_dead_jobs_requeued_total", + documentation=("Total number of dead jobs requeued"), + labelnames=("db",), + registry=_registry, +) + + +def make_metrics_server(bind_addr, port) -> HTTPServer: + infos = socket.getaddrinfo( + bind_addr, + port, + type=socket.SOCK_STREAM, + flags=socket.AI_PASSIVE, + ) + _, _, _, _, sockaddr = next(iter(infos)) + server = HTTPServer(sockaddr, MetricsHandler.factory(_registry)) + server.socket.setblocking(False) + return server diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 5cbd52ef48..42005f1d8e 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -35,7 +35,7 @@ import odoo from odoo.tools import config -from . import queue_job_config +from . import metrics, queue_job_config from .channels import ENQUEUED, NOT_DONE, RELOAD_PAYLOAD, ChannelConfig, ChannelManager SELECT_TIMEOUT = 60 @@ -166,6 +166,18 @@ def _connection_info_for(db_name): return connection_info +def _metrics_config() -> tuple[str, int | None]: + listen_address = ( + os.environ.get("ODOO_QUEUE_JOB_METRICS_LISTEN_ADDRESS") + or queue_job_config.get("metrics_listen_address") + or "127.0.0.1" + ) + port = os.environ.get("ODOO_QUEUE_JOB_METRICS_PORT") or queue_job_config.get( + "metrics_port" + ) + return listen_address, int(port) if port else None + + def _async_http_get(scheme, host, port, user, password, db_name, job_uuid): # TODO: better way to HTTP GET asynchronously (grequest, ...)? # if this was python3 I would be doing this with @@ -451,6 +463,7 @@ def requeue_dead_jobs(self): for (uuid,) in cr.fetchall(): _logger.warning("Re-queued dead job with uuid: %s", uuid) + metrics.dead_jobs_requeued_total.labels(db=self.db_name).inc() class QueueJobRunner: @@ -499,6 +512,20 @@ def __init__( self.db_by_name = {} self._stop = False self._stop_pipe = os.pipe() + metrics_listen_address, metrics_port = _metrics_config() + if metrics_listen_address and metrics_port: + self._metrics_server = metrics.make_metrics_server( + metrics_listen_address, metrics_port + ) + _logger.info( + "jobrunner metrics served on http://%s:%d/metrics", + metrics_listen_address, + metrics_port, + ) + else: + self._metrics_server = None + _logger.info("jobrunner metrics server not configured") + metrics.jobs_to_do.set_function(self.channel_manager._jobs_to_do_gauge) def __del__(self): # pylint: disable=except-pass @@ -672,6 +699,7 @@ def _all_running_count(self) -> int: def _dispatch_job(self, job): _logger.info("asking Odoo to run job %s on db %s", job.uuid, job.db_name) self.db_by_name[job.db_name].set_job_enqueued(job.uuid) + metrics.jobs_scheduled_total.labels(db=job.db_name).inc() _async_http_get( self.scheme, self.host, @@ -783,6 +811,8 @@ def wait_notification(self): # we'll select() on database connections and the stop pipe conns = [db.conn for db in self.db_by_name.values()] conns.append(self._stop_pipe[0]) + if self._metrics_server: + conns.append(self._metrics_server.fileno()) # look if the channels specify a wakeup time wakeup_time = self.next_wakeup_time() if not wakeup_time: @@ -809,7 +839,18 @@ def wait_notification(self): if key.fileobj == self._stop_pipe[0]: # stop-pipe is not a conn so doesn't need poll() continue - key.fileobj.poll() + elif ( + self._metrics_server + and key.fileobj == self._metrics_server.fileno() + ): + # TODO: Room for improvement? handle_request does a + # select() which is redundant here because we know + # the socket is ready. _handle_request_noblock() + # seems better suited but is not public. + self._metrics_server.handle_request() + else: + # db conn + key.fileobj.poll() def stop(self): _logger.info("graceful stop requested") From f799b8c7fc7361d3d98a0becf8f1a29128e59f73 Mon Sep 17 00:00:00 2001 From: Yannick Payot Date: Thu, 29 Oct 2026 17:15:18 +0100 Subject: [PATCH 2/5] [ADD] metrics: add db label to metrics This adds a 'db' label to channel gauges to track which databases have jobs in each channel. This allows for better monitoring when multiple databases share the same channel structure. --- queue_job/README.rst | 3 +++ queue_job/__manifest__.py | 2 +- queue_job/jobrunner/channels.py | 24 ++++++++++++++++-------- queue_job/jobrunner/metrics.py | 3 ++- queue_job/jobrunner/runner.py | 5 ++++- queue_job/readme/CONFIGURE.md | 4 ++-- queue_job/static/description/index.html | 3 +++ requirements.txt | 1 + 8 files changed, 32 insertions(+), 13 deletions(-) diff --git a/queue_job/README.rst b/queue_job/README.rst index 3748fa195d..2a15674433 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -213,6 +213,9 @@ The execution of channels by the job runner is defined by: ``--http-interface`` or ``localhost`` if unset - ``ODOO_QUEUE_JOB_HTTP_AUTH_USER=jobrunner``, default empty - ``ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t``, default empty + - ``ODOO_QUEUE_JOB_METRICS_LISTEN_ADDRESS=127.0.0.1``, default + 127.0.0.1 + - ``ODOO_QUEUE_JOB_METRICS_PORT=9000``, default empty - Using the Odoo configuration file (set either ``channels``, either ``max_capacity`` and/or ``db_max_capacity``) diff --git a/queue_job/__manifest__.py b/queue_job/__manifest__.py index 3a3d2d536f..434fa56940 100644 --- a/queue_job/__manifest__.py +++ b/queue_job/__manifest__.py @@ -8,7 +8,7 @@ "license": "LGPL-3", "category": "Generic Modules", "depends": ["mail", "base_sparse_field", "web"], - "external_dependencies": {"python": ["requests"]}, + "external_dependencies": {"python": ["requests", "prometheus_client"]}, "data": [ "security/security.xml", "security/ir.model.access.csv", diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index e263bf6004..ee440e9ede 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -440,7 +440,6 @@ def __init__( self.parent = parent if self.parent: self.parent.children[name] = self - self.parent._register_channel_gauges() self.children = {} self._queue = ChannelQueue() self._running = set() @@ -456,14 +455,12 @@ def __init__( self.sequential_default = sequential_default self.paused = paused self._metrics_labels = None - self._register_channel_gauges() - def __del__(self): - self._unregister_channel_gauges() - - def _register_channel_gauges(self) -> None: - self._unregister_channel_gauges() + def register_metrics(self, db_name) -> None: + """Register gauges for a specific database in this channel.""" + self.unregister_metrics() self._metrics_labels = { + "db": db_name, "channel": self.fullname, "root": not bool(self.parent), "leaf": not bool(self.children), @@ -484,7 +481,7 @@ def _register_channel_gauges(self) -> None: **self._metrics_labels ).set_function(weakref.proxy(self)._waiting_dependencies_gauge) - def _unregister_channel_gauges(self) -> None: + def unregister_metrics(self) -> None: if not self._metrics_labels: return metrics.channel_capacity.remove_by_labels(self._metrics_labels) @@ -492,6 +489,7 @@ def _unregister_channel_gauges(self) -> None: metrics.channel_running.remove_by_labels(self._metrics_labels) metrics.channel_failed.remove_by_labels(self._metrics_labels) metrics.channel_waiting_dependencies.remove_by_labels(self._metrics_labels) + self._metrics_labels = None def _capacity_gauge(self) -> float: if self.capacity is None: @@ -1284,3 +1282,13 @@ def running_count(self) -> int: def _jobs_to_do_gauge(self) -> float: return len(self._jobs_by_uuid) + + def register_metrics(self, db_name): + for channel in self._channels_by_name.values(): + channel.register_metrics(db_name) + metrics.jobs_to_do.labels({"db": db_name}).set_function(self._jobs_to_do_gauge) + + def unregister_metrics(self, db_name): + for channel in self._channels_by_name.values(): + channel.unregister_metrics() + metrics.jobs_to_do.remove_by_labels({"db": db_name}) diff --git a/queue_job/jobrunner/metrics.py b/queue_job/jobrunner/metrics.py index f52020699f..1deb8a2e2d 100644 --- a/queue_job/jobrunner/metrics.py +++ b/queue_job/jobrunner/metrics.py @@ -5,7 +5,7 @@ _registry = CollectorRegistry() -_channel_label_names = ("channel", "root", "leaf") +_channel_label_names = ("db", "channel", "root", "leaf") channel_capacity = Gauge( "queue_job_channel_capacity", documentation="Channel Capacity", @@ -42,6 +42,7 @@ documentation=( "Number of jobs waiting to be done (including running and failed jobs)" ), + labelnames=("db",), registry=_registry, ) jobs_scheduled_total = Counter( diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 42005f1d8e..b933add3ef 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -492,6 +492,7 @@ def __init__( channel_manager = ChannelManager() channel_manager.simple_configure(channel_config_string) self._server_wide_channel_manager = channel_manager + self._server_wide_channel_manager.register_metrics(db_name=None) # max_capacity is always equal to the root channel in server-wide # configuration max_capacity = channel_manager.get_channel_by_name("root").capacity @@ -525,7 +526,6 @@ def __init__( else: self._metrics_server = None _logger.info("jobrunner metrics server not configured") - metrics.jobs_to_do.set_function(self.channel_manager._jobs_to_do_gauge) def __del__(self): # pylint: disable=except-pass @@ -662,10 +662,13 @@ def _configure_db(self, db): """Build the channel manager for a database and reload its jobs""" db_name = db.db_name channel_manager = self._build_channel_manager(db) + if db_name in self._channel_manager_by_db: + self._channel_manager_by_db[db_name].unregister_metrics(db_name) self._channel_manager_by_db[db_name] = channel_manager self._channel_managers = list(self._channel_manager_by_db.values()) self._notify_existing_jobs(db, channel_manager) _logger.info("channels configuration loaded for db %s", db_name) + channel_manager.register_metrics(db_name) def _notify_existing_jobs(self, db, channel_manager): with db.select_jobs("state in %s", (NOT_DONE,)) as cr: diff --git a/queue_job/readme/CONFIGURE.md b/queue_job/readme/CONFIGURE.md index d2233d54b8..a5c110efde 100644 --- a/queue_job/readme/CONFIGURE.md +++ b/queue_job/readme/CONFIGURE.md @@ -65,7 +65,6 @@ The execution of channels by the job runner is defined by: - `throttle`: minimum delay, in seconds, between the start of two jobs - `paused`: stop running jobs in this channel and its subchannels - **Job Runner Configuration Parameters** - Using environment variables: @@ -82,6 +81,8 @@ The execution of channels by the job runner is defined by: or `localhost` if unset - `ODOO_QUEUE_JOB_HTTP_AUTH_USER=jobrunner`, default empty - `ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t`, default empty + - `ODOO_QUEUE_JOB_METRICS_LISTEN_ADDRESS=127.0.0.1`, default 127.0.0.1 + - `ODOO_QUEUE_JOB_METRICS_PORT=9000`, default empty - Using the Odoo configuration file (set either `channels`, either `max_capacity` and/or `db_max_capacity`) @@ -133,7 +134,6 @@ server_wide_modules = web,queue_job [^1]: It works with the threaded Odoo server too, although this way of running Odoo is obviously not for production purposes. - **Migrating from server-wide channels to per-database channels** As long as `channels` (or `ODOO_QUEUE_JOB_CHANNELS`) is set, the job diff --git a/queue_job/static/description/index.html b/queue_job/static/description/index.html index cf6c20a4b8..e223c22624 100644 --- a/queue_job/static/description/index.html +++ b/queue_job/static/description/index.html @@ -558,6 +558,9 @@

Configuration

--http-interface or localhost if unset
  • ODOO_QUEUE_JOB_HTTP_AUTH_USER=jobrunner, default empty
  • ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t, default empty
  • +
  • ODOO_QUEUE_JOB_METRICS_LISTEN_ADDRESS=127.0.0.1, default +127.0.0.1
  • +
  • ODOO_QUEUE_JOB_METRICS_PORT=9000, default empty
  • diff --git a/requirements.txt b/requirements.txt index b4d39fb9e0..26cd22456a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ # generated from manifests external_dependencies +prometheus_client requests From fc613354b948d04e21accd22be6eb27b73326ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 5 Nov 2026 19:19:45 +0100 Subject: [PATCH 3/5] [REM] metrics: remove leaf and root labels Not sure that's useful. Let's wait for the actual use case. --- queue_job/jobrunner/channels.py | 2 -- queue_job/jobrunner/metrics.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index ee440e9ede..9b7b775ac7 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -462,8 +462,6 @@ def register_metrics(self, db_name) -> None: self._metrics_labels = { "db": db_name, "channel": self.fullname, - "root": not bool(self.parent), - "leaf": not bool(self.children), } metrics.channel_capacity.labels(**self._metrics_labels).set_function( weakref.proxy(self)._capacity_gauge diff --git a/queue_job/jobrunner/metrics.py b/queue_job/jobrunner/metrics.py index 1deb8a2e2d..709188a8bf 100644 --- a/queue_job/jobrunner/metrics.py +++ b/queue_job/jobrunner/metrics.py @@ -5,7 +5,7 @@ _registry = CollectorRegistry() -_channel_label_names = ("db", "channel", "root", "leaf") +_channel_label_names = ("db", "channel") channel_capacity = Gauge( "queue_job_channel_capacity", documentation="Channel Capacity", From 405cbc28d4a34d11191c0cf5f7be637c9df776a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 9 Nov 2026 16:10:14 +0100 Subject: [PATCH 4/5] [IMP] metrics: start metrics server after acquiring leader lock --- queue_job/jobrunner/runner.py | 41 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index b933add3ef..a18b0f693f 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -513,19 +513,7 @@ def __init__( self.db_by_name = {} self._stop = False self._stop_pipe = os.pipe() - metrics_listen_address, metrics_port = _metrics_config() - if metrics_listen_address and metrics_port: - self._metrics_server = metrics.make_metrics_server( - metrics_listen_address, metrics_port - ) - _logger.info( - "jobrunner metrics served on http://%s:%d/metrics", - metrics_listen_address, - metrics_port, - ) - else: - self._metrics_server = None - _logger.info("jobrunner metrics server not configured") + self._metrics_server = None def __del__(self): # pylint: disable=except-pass @@ -689,6 +677,27 @@ def initialize_databases(self): else: db.close() + def initialize_metrics_server(self): + metrics_listen_address, metrics_port = _metrics_config() + if metrics_listen_address and metrics_port: + self._metrics_server = metrics.make_metrics_server( + metrics_listen_address, metrics_port + ) + _logger.info( + "jobrunner metrics served on http://%s:%d/metrics", + metrics_listen_address, + metrics_port, + ) + else: + self._metrics_server = None + _logger.info("jobrunner metrics server not configured") + + def shutdown_metrics_server(self): + if not self._metrics_server: + return + self._metrics_server.server_close() + self._metrics_server = None + def requeue_dead_jobs(self): for db in self.db_by_name.values(): if db.has_queue_job: @@ -866,11 +875,17 @@ def run(self): while not self._stop: # outer loop does exception recovery try: + # shutdown metrics server if one exists, because if we loose + # the leader election, we don't want it to listen and never respond + self.shutdown_metrics_server() _logger.debug("initializing database connections") # TODO: how to detect new databases or databases # on which queue_job is installed after server start? self.initialize_databases() _logger.info("database connections ready") + # at this point we have acquired the leader lock, so we can + # start the metrics server + self.initialize_metrics_server() # inner loop does the normal processing while not self._stop: self.requeue_dead_jobs() From db858babb6cbddedb6380ecbfe240f7511396986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Mon, 9 Nov 2026 16:30:40 +0100 Subject: [PATCH 5/5] [IMP] metrics: use empty value instead of 'None' for db label --- queue_job/jobrunner/channels.py | 8 +++++--- queue_job/jobrunner/metrics.py | 4 ++++ queue_job/jobrunner/runner.py | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index 9b7b775ac7..37ea687b03 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -460,7 +460,7 @@ def register_metrics(self, db_name) -> None: """Register gauges for a specific database in this channel.""" self.unregister_metrics() self._metrics_labels = { - "db": db_name, + "db": metrics.db_label(db_name), "channel": self.fullname, } metrics.channel_capacity.labels(**self._metrics_labels).set_function( @@ -1284,9 +1284,11 @@ def _jobs_to_do_gauge(self) -> float: def register_metrics(self, db_name): for channel in self._channels_by_name.values(): channel.register_metrics(db_name) - metrics.jobs_to_do.labels({"db": db_name}).set_function(self._jobs_to_do_gauge) + metrics.jobs_to_do.labels(db=metrics.db_label(db_name)).set_function( + self._jobs_to_do_gauge + ) def unregister_metrics(self, db_name): for channel in self._channels_by_name.values(): channel.unregister_metrics() - metrics.jobs_to_do.remove_by_labels({"db": db_name}) + metrics.jobs_to_do.remove_by_labels({"db": metrics.db_label(db_name)}) diff --git a/queue_job/jobrunner/metrics.py b/queue_job/jobrunner/metrics.py index 709188a8bf..a60797d885 100644 --- a/queue_job/jobrunner/metrics.py +++ b/queue_job/jobrunner/metrics.py @@ -61,6 +61,10 @@ ) +def db_label(db_name: str | None) -> str: + return db_name or "" + + def make_metrics_server(bind_addr, port) -> HTTPServer: infos = socket.getaddrinfo( bind_addr, diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index a18b0f693f..0910a950b9 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -463,7 +463,7 @@ def requeue_dead_jobs(self): for (uuid,) in cr.fetchall(): _logger.warning("Re-queued dead job with uuid: %s", uuid) - metrics.dead_jobs_requeued_total.labels(db=self.db_name).inc() + metrics.dead_jobs_requeued_total.labels(db=metrics.db_label(self.db_name)).inc() class QueueJobRunner: @@ -711,7 +711,7 @@ def _all_running_count(self) -> int: def _dispatch_job(self, job): _logger.info("asking Odoo to run job %s on db %s", job.uuid, job.db_name) self.db_by_name[job.db_name].set_job_enqueued(job.uuid) - metrics.jobs_scheduled_total.labels(db=job.db_name).inc() + metrics.jobs_scheduled_total.labels(db=metrics.db_label(job.db_name)).inc() _async_http_get( self.scheme, self.host,