From b9a625e5c98ad0b58ee0ede79d92c686eb0f4f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Tue, 19 May 2026 16:40:34 +0200 Subject: [PATCH 01/14] Add full machine monitorization --- package/CMakeLists.txt | 1 + package/src/machmon.cpp | 101 ++++++++++++++++++++++++++++++++++++++++ package/src/machmon.h | 55 ++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 package/src/machmon.cpp create mode 100644 package/src/machmon.h diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt index 0c8f24d..8e36438 100644 --- a/package/CMakeLists.txt +++ b/package/CMakeLists.txt @@ -10,6 +10,7 @@ add_executable(prmon src/prmon.cpp src/countmon.cpp src/wallmon.cpp src/memmon.cpp + src/machmon.cpp ) target_link_libraries(prmon PRIVATE diff --git a/package/src/machmon.cpp b/package/src/machmon.cpp new file mode 100644 index 0000000..e196588 --- /dev/null +++ b/package/src/machmon.cpp @@ -0,0 +1,101 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +#include "machmon.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +#define MONITOR_NAME "machmon" + +// Constructor; uses RAII pattern to be valid after construction +machmon::machmon() : mach_stats{} { + log_init(MONITOR_NAME); +#undef MONITOR_NAME + for (const auto& param : params) { + mach_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); + } +} + +// Read the first line of /proc/stat and return {utime, stime, ntime} +// Layout of /proc/stat first line: +// cpu user nice system idle iowait irq softirq ... +// [1] [2] [3] [4] ... +static std::tuple read_proc_stat( + const std::string& read_path) { + std::string stat_path = "/proc/stat"; + std::ifstream proc_stat{stat_path}; + if (!proc_stat) + throw std::runtime_error("Cannot open " + stat_path); + + std::string cpu_label{}; + long user{}, nice{}, system{}; + + // First line: "cpu user nice system idle ..." + if (!(proc_stat >> cpu_label >> user >> nice >> system)) + throw std::runtime_error("Unexpected format in " + stat_path); + + return {user, system, nice}; +} + +void machmon::update_stats(const std::vector& /*pids*/, + const std::string read_path) { + long utime, stime, ntime; + std::tie(utime, stime, ntime) = read_proc_stat(read_path); + + // Subtract initial snapshot on first read + if (!initialised) { + initial_utime = utime; + initial_stime = stime; + initial_ntime = ntime; + initialised = true; + } + + const long clk = sysconf(_SC_CLK_TCK); + mach_stats.at("utime_total").set_value((utime - initial_utime) / clk); + mach_stats.at("stime_total").set_value((stime - initial_stime) / clk); + mach_stats.at("ntime_total").set_value((ntime - initial_ntime) / clk); + +} + +// Return the summed counters +prmon::monitored_value_map const machmon::get_text_stats() { + prmon::monitored_value_map cpu_stat_map{}; + for (const auto& value : mach_stats) { + cpu_stat_map[value.first] = value.second.get_value(); + } + return cpu_stat_map; +} + +// Same for JSON +prmon::monitored_value_map const machmon::get_json_total_stats() { + return machmon::get_text_stats(); +} + +// No averages for global CPU time either +prmon::monitored_average_map const machmon::get_json_average_stats( + unsigned long long /*elapsed_clock_ticks*/) { + static const prmon::monitored_average_map empty_average_stats{}; + return empty_average_stats; +} + +// Return the parameter list +prmon::parameter_list const machmon::get_parameter_list() { return params; } + +// No extra hardware info needed here +void const machmon::get_hardware_info(nlohmann::json& /*hw_json*/) { + return; +} + +void const machmon::get_unit_info(nlohmann::json& unit_json) { + prmon::fill_units(unit_json, params); + return; +} \ No newline at end of file diff --git a/package/src/machmon.h b/package/src/machmon.h new file mode 100644 index 0000000..387d241 --- /dev/null +++ b/package/src/machmon.h @@ -0,0 +1,55 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +// Full CPU monitoring class +// + +#pragma once + +#ifndef PRMON_MACHMON_H +#define PRMON_MACHMON_H 1 + +#include +#include +#include +#include + +#include "Imonitor.h" +#include "MessageBase.h" +#include "parameter.h" +#include "registry.h" + +class machmon final : public Imonitor, public MessageBase { + private: + const prmon::parameter_list params = { + {"utime_total", "s", "Total user CPU time from /proc/stat"}, + {"stime_total", "s", "Total system CPU time from /proc/stat"}, + {"ntime_total", "s", "Total nice CPU time from /proc/stat"}, + }; + + prmon::monitored_list mach_stats; // era monitored_value_map + + long initial_utime{0L}; + long initial_stime{0L}; + long initial_ntime{0L}; + bool initialised{false}; + + public: + machmon(); + + void update_stats(const std::vector& pids, + const std::string read_path = ""); + + prmon::monitored_value_map const get_text_stats(); + prmon::monitored_value_map const get_json_total_stats(); + prmon::monitored_average_map const get_json_average_stats( + unsigned long long elapsed_clock_ticks); + prmon::parameter_list const get_parameter_list(); + void const get_hardware_info(nlohmann::json& hw_json); + void const get_unit_info(nlohmann::json& unit_json); + bool const is_valid() { return true; } +}; +REGISTER_MONITOR(Imonitor, machmon, "Monitor full machine cpu time") + + +#endif // PRMON_MACHMON_H \ No newline at end of file From 6c74ca1662123a20eee132bb84cc9a6e68e9e182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Wed, 20 May 2026 15:45:11 +0200 Subject: [PATCH 02/14] Add curlmon, add curl library --- package/CMakeLists.txt | 4 ++ package/src/curlmon.cpp | 138 ++++++++++++++++++++++++++++++++++++++++ package/src/curlmon.h | 60 +++++++++++++++++ package/src/machmon.cpp | 4 +- 4 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 package/src/curlmon.cpp create mode 100644 package/src/curlmon.h diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt index 8e36438..1c9eeb8 100644 --- a/package/CMakeLists.txt +++ b/package/CMakeLists.txt @@ -11,14 +11,18 @@ add_executable(prmon src/prmon.cpp src/wallmon.cpp src/memmon.cpp src/machmon.cpp + src/curlmon.cpp ) target_link_libraries(prmon PRIVATE nlohmann_json::nlohmann_json spdlog::spdlog_header_only + CURL::libcurl dl ) +find_package(CURL REQUIRED) + if (BUILD_BENCHMARK_LOG) add_executable(benchmark-log benchmarks/benchmark-log.cpp) endif(BUILD_BENCHMARK_LOG) diff --git a/package/src/curlmon.cpp b/package/src/curlmon.cpp new file mode 100644 index 0000000..b100219 --- /dev/null +++ b/package/src/curlmon.cpp @@ -0,0 +1,138 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +#include "curlmon.h" + +#include + +#include +#include +#include +#include +#include + +#include "utils.h" + +#define MONITOR_NAME "curlmon" + +// libcurl write, appending data to a std::string +static size_t curl_write_cb(char* ptr, size_t size, size_t nmemb, + void* userdata) { + auto* buf = reinterpret_cast(userdata); + buf->append(ptr, size * nmemb); + return size * nmemb; +} + + +// Constructor; uses RAII pattern to be valid after construction +curlmon::curlmon() : curl_stats{} { + log_init(MONITOR_NAME); +#undef MONITOR_NAME + for (const auto& param : params) { + curl_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); + } +} + +// Fetch Prometheus text from the ctt URL +std::string curlmon::fetch_metrics(const std::string& url) const { + std::string response{}; + CURL* curl = curl_easy_init(); + if (!curl) throw std::runtime_error("Failed to initialise libcurl"); + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); + + CURLcode res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) + throw std::runtime_error(std::string("curl request failed: ") + + curl_easy_strerror(res)); + return response; +} + +// Parse a single Prometheus line: "metric_name{labels} value timestamp" +// Returns the value if the line starts with metric_name, otherwise -1.0 +double curlmon::parse_metric_line(const std::string& line, + const std::string& metric_name) const { + if (line.empty() || line[0] == '#') return -1.0; + + if (line.rfind(metric_name, 0) != 0) return -1.0; + + char next = line[metric_name.size()]; + if (next != '{' && next != ' ' && next != '\t') return -1.0; + + size_t value_start = line.find('}'); + if (value_start == std::string::npos) + value_start = metric_name.size(); + else + value_start += 1; + + while (value_start < line.size() && std::isspace(line[value_start])) + ++value_start; + + try { + return std::stod(line.substr(value_start)); + } catch (...) { + return -1.0; + } +} + +void curlmon::update_stats(const std::vector& /*pids*/, + const std::string read_path) { + std::unordered_map accumulated{}; + for (const auto& param : params) accumulated[param.get_name()] = 0.0; + + for (const auto& url : metrics_urls) { + try { + const std::string body = fetch_metrics(url); + std::istringstream stream{body}; + std::string line{}; + + while (std::getline(stream, line)) { + for (const auto& param : params) { + double val = parse_metric_line(line, param.get_name()); + if (val >= 0.0) accumulated[param.get_name()] += val; + } + } + } catch (const std::exception& e) { + error(std::string("Error fetching metrics from ") + url + ": " + e.what()); + } + } + + for (const auto& param : params) { + auto val = static_cast(accumulated[param.get_name()] * 1000.0); + curl_stats.at(param.get_name()).set_value(val); + } +} + + +// Return the monitored values +prmon::monitored_value_map const curlmon::get_text_stats() { + prmon::monitored_value_map stat_map{}; + for (const auto& value : curl_stats) { + stat_map[value.first] = value.second.get_value(); + } + return stat_map; +} + +prmon::monitored_value_map const curlmon::get_json_total_stats() { + return curlmon::get_text_stats(); +} + +prmon::monitored_average_map const curlmon::get_json_average_stats( + unsigned long long /*elapsed_clock_ticks*/) { + static const prmon::monitored_average_map empty{}; + return empty; +} + +prmon::parameter_list const curlmon::get_parameter_list() { return params; } + +void const curlmon::get_hardware_info(nlohmann::json& /*hw_json*/) { return; } + +void const curlmon::get_unit_info(nlohmann::json& unit_json) { + prmon::fill_units(unit_json, params); + return; +} \ No newline at end of file diff --git a/package/src/curlmon.h b/package/src/curlmon.h new file mode 100644 index 0000000..4c7d702 --- /dev/null +++ b/package/src/curlmon.h @@ -0,0 +1,60 @@ +// Copyright (C) 2018-2025 CERN +// License Apache2 - see LICENCE file + +// Curl to prometheus monitoring class +// + +#pragma once + +#ifndef PRMON_CURLMON_H +#define PRMON_CURLMON_H 1 + +#include +#include +#include +#include + +#include "Imonitor.h" +#include "MessageBase.h" +#include "parameter.h" +#include "registry.h" + +class curlmon final : public Imonitor, public MessageBase { + private: + const prmon::parameter_list params = { + {"ipmi_dcmi_power_consumption_watts", "W", "DCMI power consumption"}, + {"ipmi_power_watts", "W", "IPMI power consumption"}, + {"node_hwmon_power_watt", "W", "HWMON power consumption"}, + {"node_rapl_package_joules_total", "W", "RAPL power consumption"}, + }; + + prmon::monitored_list curl_stats; // era monitored_value_map + + const std::vector metrics_urls{ + "http://localhost:9100/metrics", + "http://localhost:9290/metrics" + }; + + std::string fetch_metrics(const std::string& url) const; + double parse_metric_line(const std::string& line, + const std::string& metric_name) const; + + public: + curlmon(); + + void update_stats(const std::vector& pids, + const std::string read_path = ""); + + prmon::monitored_value_map const get_text_stats(); + prmon::monitored_value_map const get_json_total_stats(); + prmon::monitored_average_map const get_json_average_stats( + unsigned long long elapsed_clock_ticks); + prmon::parameter_list const get_parameter_list(); + void const get_hardware_info(nlohmann::json& hw_json); + void const get_unit_info(nlohmann::json& unit_json); + bool const is_valid() { return true; } +}; +REGISTER_MONITOR(Imonitor, curlmon, "Monitor power consumption - prometheus") + + +#endif // PRMON_CURLMON_H \ No newline at end of file diff --git a/package/src/machmon.cpp b/package/src/machmon.cpp index e196588..b9e169c 100644 --- a/package/src/machmon.cpp +++ b/package/src/machmon.cpp @@ -26,9 +26,7 @@ machmon::machmon() : mach_stats{} { } // Read the first line of /proc/stat and return {utime, stime, ntime} -// Layout of /proc/stat first line: -// cpu user nice system idle iowait irq softirq ... -// [1] [2] [3] [4] ... + static std::tuple read_proc_stat( const std::string& read_path) { std::string stat_path = "/proc/stat"; From c7acb26c44bf5021a357ff5f448d8f171b28e5c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Mon, 1 Jun 2026 14:45:10 +0200 Subject: [PATCH 03/14] Improve curlmon: 2 counters for RAPL, stop warnings and disable if not working --- package/src/curlmon.cpp | 97 +++++++++++++++++++++++++++++++++++------ package/src/curlmon.h | 13 +++--- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/package/src/curlmon.cpp b/package/src/curlmon.cpp index b100219..1fb9d78 100644 --- a/package/src/curlmon.cpp +++ b/package/src/curlmon.cpp @@ -80,31 +80,102 @@ double curlmon::parse_metric_line(const std::string& line, } } + void curlmon::update_stats(const std::vector& /*pids*/, - const std::string read_path) { - std::unordered_map accumulated{}; - for (const auto& param : params) accumulated[param.get_name()] = 0.0; + const std::string /*read_path*/) { + // Collect matching lines from all URLs + std::vector dcmi_lines{}; + std::vector rapl_lines{}; + std::vector ipmi_lines{}; + std::vector hwmon_lines{}; + + if (connection_failed) return; + int failed_urls = 0; for (const auto& url : metrics_urls) { try { const std::string body = fetch_metrics(url); std::istringstream stream{body}; std::string line{}; - while (std::getline(stream, line)) { - for (const auto& param : params) { - double val = parse_metric_line(line, param.get_name()); - if (val >= 0.0) accumulated[param.get_name()] += val; - } + std::string lower = line; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + if (lower.find("ipmi_dcmi_power_consumption_watts") != std::string::npos) + dcmi_lines.push_back(line); + if (lower.find("node_rapl_package_joules_total") != std::string::npos) + rapl_lines.push_back(line); + if (lower.find("ipmi_power_watts") != std::string::npos) + ipmi_lines.push_back(line); + if (lower.find("node_hwmon_power_watt") != std::string::npos) + hwmon_lines.push_back(line); } - } catch (const std::exception& e) { - error(std::string("Error fetching metrics from ") + url + ": " + e.what()); + } catch (...) { + ++failed_urls; // Silencioso: no se loga nada por URL individual } } - for (const auto& param : params) { - auto val = static_cast(accumulated[param.get_name()] * 1000.0); - curl_stats.at(param.get_name()).set_value(val); + if (failed_urls == (int)metrics_urls.size()) { + error("Cannot connect to any exporter endpoint, disabling curlmon"); + connection_failed = true; + return; + } + + // DCMI: sum all matching lines + double dcmi = 0.0; + try { + for (const auto& line : dcmi_lines) { + double val = parse_metric_line(line, "ipmi_dcmi_power_consumption_watts"); + if (val >= 0.0) dcmi += val; + } + } catch (...) { dcmi = 0.0; } + + // RAPL: penultimate line -> rapl_1, last line -> rapl_2 + double rapl_1 = 0.0, rapl_2 = 0.0; + try { + if (rapl_lines.size() >= 2) { + double v1 = parse_metric_line(rapl_lines[rapl_lines.size() - 2], + "node_rapl_package_joules_total"); + double v2 = parse_metric_line(rapl_lines[rapl_lines.size() - 1], + "node_rapl_package_joules_total"); + if (v1 >= 0.0) rapl_1 = v1; + if (v2 >= 0.0) rapl_2 = v2; + } else if (rapl_lines.size() == 1) { + double v = parse_metric_line(rapl_lines[0], "node_rapl_package_joules_total"); + if (v >= 0.0) rapl_1 = v; + } + } catch (...) { rapl_1 = rapl_2 = 0.0; } + + // IPMI: last matching line + double ipmi = 0.0; + try { + if (!ipmi_lines.empty()) { + double val = parse_metric_line(ipmi_lines.back(), "ipmi_power_watts"); + if (val >= 0.0) ipmi = val; + } + } catch (...) { ipmi = 0.0; } + + // HWMON: last matching line + double hwmon = 0.0; + try { + if (!hwmon_lines.empty()) { + double val = parse_metric_line(hwmon_lines.back(), "node_hwmon_power_average_watt"); + if (val >= 0.0) hwmon = val; + } + } catch (...) { hwmon = 0.0; } + + try { + curl_stats.at("dcmi").set_value( + static_cast(dcmi)); + curl_stats.at("rapl_1").set_value( + static_cast(rapl_1)); + curl_stats.at("rapl_2").set_value( + static_cast(rapl_2)); + curl_stats.at("ipmi").set_value( + static_cast(ipmi)); + curl_stats.at("hwmon").set_value( + static_cast(hwmon)); + } catch (const std::exception& e) { + error(std::string("Error storing metrics: ") + e.what()); } } diff --git a/package/src/curlmon.h b/package/src/curlmon.h index 4c7d702..b5260f7 100644 --- a/package/src/curlmon.h +++ b/package/src/curlmon.h @@ -22,13 +22,16 @@ class curlmon final : public Imonitor, public MessageBase { private: const prmon::parameter_list params = { - {"ipmi_dcmi_power_consumption_watts", "W", "DCMI power consumption"}, - {"ipmi_power_watts", "W", "IPMI power consumption"}, - {"node_hwmon_power_watt", "W", "HWMON power consumption"}, - {"node_rapl_package_joules_total", "W", "RAPL power consumption"}, + {"dcmi", "W", "DCMI power consumption"}, + {"ipmi", "W", "IPMI power consumption"}, + {"hwmon", "W", "HWMON power consumption"}, + {"rapl_1", "J", "RAPL package 1 energy in joules"}, + {"rapl_2", "J", "RAPL package 2 energy in joules"} }; + + bool connection_failed{false}; // To disable monitor if all ports fail - prmon::monitored_list curl_stats; // era monitored_value_map + prmon::monitored_list curl_stats; const std::vector metrics_urls{ "http://localhost:9100/metrics", From 2fef3ae0a0595a7ebe860ab50b71e30f6f9e86b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Thu, 8 Oct 2026 11:02:47 +0200 Subject: [PATCH 04/14] Clean up and update README, improve comments on curlmon.cpp. --- README.md | 326 +--------------------------------------- package/src/curlmon.cpp | 6 +- 2 files changed, 10 insertions(+), 322 deletions(-) diff --git a/README.md b/README.md index 7c5bc28..bca3a05 100644 --- a/README.md +++ b/README.md @@ -3,325 +3,11 @@ [![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-) -[![Build Status][build-img]][build-link] [![License][license-img]][license-url] [![Codefactor][codefactor-img]][codefactor-url] [![HSF Project][hsf-project-img]][hsf-project-url] +This fork of prmon aims to test two additional collectors (curlmon and machmon) that can add information regarding power consumption. The idea stems from a proposal to improve job power consumption monitoring via an estimation of the usage of the machine by a given job. -[build-img]: https://github.com/hsf/prmon/actions/workflows/ci.yml/badge.svg?branch=main -[build-link]: https://github.com/hsf/prmon/actions/workflows/ci.yml?query=branch%3Amain -[license-img]: https://img.shields.io/github/license/hsf/prmon.svg -[license-url]: https://github.com/hsf/prmon/blob/main/LICENSE -[codefactor-img]: https://www.codefactor.io/repository/github/HSF/prmon/badge -[codefactor-url]: https://www.codefactor.io/repository/github/HSF/prmon -[hsf-project-img]: https://raw.githubusercontent.com/HSF/hsf.github.io/refs/heads/main/images/HSF-logo/HSF-Affiliated.svg -[hsf-project-url]: https://hepsoftwarefoundation.org/projects/affiliated.html +# machmon +Similarly to cpumon, this collector reads the cpu times of the whole machine (utime, stime and ntime). By comparing pidtimes with machtimes, we can compute a ratio that estimates the fraction of the machine that is being used by the monitored process. +$f_{usage} = \frac{t_{PID}}{t_{total}}$ -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2554202.svg)](https://doi.org/10.5281/zenodo.2554202) - -The PRocess MONitor is a small stand alone program that can monitor -the resource consumption of a process and its children. This is -useful in the context of the WLCG/HSF working group to evaluate -the costs and performance of HEP workflows in WLCG. In a previous -incarnation (MemoryMonitor) it has been used by ATLAS for some time to -gather data on resource consumption by production jobs. One of its -most useful features is to use smaps to correctly calculate the -*Proportional Set Size* in the group of processes monitored, which -is a much better indication of the true memory consumption of -a group of processes where children share many pages. - -prmon currently runs on Linux machines as it requires access to the -`/proc` interface to process statistics. - -## Build and Deployment - -### Cloning the project - -As prmon has dependencies on submodules, clone the project as - - git clone --recurse-submodules https://github.com/HSF/prmon.git - -### Building the project - -Building prmon requires a C++ compiler that fully supports C++11, -and CMake version 3.10 or higher. It also has dependencies on: - - - [Niels Lohmann JSON libraries](https://github.com/nlohmann/json) - - `nlohmann-json-dev` in Ubuntu 18, `nlohmann-json3-dev` in Ubuntu 20 - - [spdlog: Fast C++ logging library](https://github.com/gabime/spdlog) - - `libspdlog-dev` in Ubuntu - -and can use either external system-supplied versions or internal copies -provided by submodules. - -Building is usually as simple as: - - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX= -S .. -B . - make -j - make install - -Unless otherwise specified, the default behavior for dependencies is to first -try to find an external version and fall back to the internal submodule copy -if not found. To explicitly force the use of either add any of the following -configure options: - - `-DUSE_EXTERNAL_NLOHMANN_JSON={TRUE,FALSE,`**`AUTO`**`}` - - `ON`, `TRUE`: Force an external version and fail if not found. - - `OFF`, `FALSE`: Require the internal copy be used. - - **`AUTO`**: Search for an external version and fall back to the - internal copy if not found. - - `-Dnlohmann_json_DIR=/path/to/config` - - The path to the directory containing `nlohmann_jsonConfig.cmake`. - Necessary if nlohmann_json is not installed into CMake's search path. - - `-DUSE_EXTERNAL_SPDLOG={TRUE,FALSE,`**`AUTO`**`}` - - `ON`, `TRUE`: Force an external version and fail if not found. - - `OFF`, `FALSE`: Require the internal copy be used. - - **`AUTO`**: Search for an external version and fall back to the - internal copy if not found. - - `-Dspdlog_DIR=/path/to/config` - - The path to the directory containing `spdlogConfig.cmake`. - Necessary if spdlog is not installed into CMake's search path. - -The option `-DCMAKE_BUILD_TYPE` can switch between all of the standard -build types. The default is `Release`; use `RelWithDebInfo` if you want -debug symbols. - -To build a statically linked version of `prmon`, set the `BUILD_STATIC` -CMake variable to `ON` (e.g., adding `-DBUILD_STATIC=ON` to the -command line). - -Note that in a build environment with CVMFS available the C++ compiler -and CMake can be taken by setting up a recent LCG release. - -To enable pulling and building the gtest framework as well as tests dependent on gtest, -build with `-DBUILD_GTESTS=ON`. - -### Creating a package with CPack - -A cpack based package can be created by invoking - - make package - -### Running the tests - -To run the tests of the project, first build it and then invoke - - make test - -Running the tests requires Python version 3.6 or higher. - -## Running - -The `prmon` binary is invoked with the following arguments: - -```sh -prmon [--pid PPP] [--filename prmon.txt] [--json-summary prmon.json] \ - [--log-filename prmon.log] [--interval 30] \ - [--suppress-hw-info] [--units] [--netdev DEV] \ - [--disable MON1] [--level LEV] [--level MON:LEV] \ - [--fast-memmon] \ - [-- prog arg arg ...] -``` - -* `--pid` the 'mother' PID to monitor (all children in the same process tree are monitored as well) -* `--filename` output file for time-stamped monitored values -* `--json-summary` output file for summary data written in JSON format -* `--log-filename` output file for log messages -* `--interval` time, in seconds, between monitoring snapshots -* `--suppress-hw-info` flag that turns-off hardware information collection -* `--units` add information on units for each metric to JSON file -* `--netdev` restricts network statistics to one (or more) network devices -* `--disable` is used to disable specific monitors (and can be specified multiple times); - the default is that `prmon` monitors everything that it can - * Note that the `wallmon` monitor is the only monitor that cannot be disabled -* `--level` is used to set the logging level for monitors - * `--level LEV` sets the level for all monitors to LEV - * `--level MON:LEV` sets the level for monitor MON to LEV - * The valid levels are `trace`, `debug`, `info`, `warn`, `error`, `critical` -* `--fast-memmon` toggles on fast memory monitoring using `smaps_rollup` -* `--` after this argument the following arguments are treated as a program to invoke - and remaining arguments are passed to it; `prmon` will then monitor this process - instead of being given a PID via `--pid` - -`prmon` will exit with `1` if there is a problem with inconsistent or -incomplete arguments. If `prmon` starts a program itself (using `--`) then -`prmon` will exit with the exit code of the child process. - -When invoked with `-h` or `--help` usage information is printed, as well as a -list of all available monitoring components. - -### Fast Memory Monitoring - -When invoked with `--fast-memmon` `prmon` uses the `smaps_rollup` files -that contain pre-summed memory information for each monitored process. -This is a faster approach compared to the default behavior, -where `prmon` aggregates the results itself by going over each of the monitored -processes' mappings one by one. - -If the current kernel doesn't support `smaps_rollup` then the default -approach is used. Users should also note that fast memory monitoring -might not contain all metrics that the default approach supports, e.g., -`vmem`. In that case, the missing metric will be omitted in the output. -If any of these issues are encountered, a relevant message is printed -to notify the user. - -### NVIDIA GPU Monitoring - -When NVIDIA GPUs are present, `prmon` will preferentially use the -NVIDIA Management Library (NVML) for direct GPU monitoring via -`libnvidia-ml.so`. If NVML is not available or fails to initialise, -`prmon` falls back to parsing the output of `nvidia-smi`. -No extra build-time dependencies are needed: NVML symbols are loaded -at runtime via `dlopen`, so `prmon` can be compiled on machines without -NVIDIA drivers and will detect GPU support when run on a machine that -has them. - -### Environment Variables - -The `PRMON_DISABLE_MONITOR` environment variable can be used to specify a comma -separated list of monitor names that will be disabled. This is useful when -`prmon` is being invoked by some other part of a job or workflow, so the user -does not have direct access to the command line options used. e.g. - -```sh -export PRMON_DISABLE_MONITOR=nvidiamon -other_code_that_invokes_prmon -... -``` - -Disables the `nvidiamon` monitor. - -## Outputs - -In the `filename` output file, plain text with statistics written every -`interval` seconds are written. The first line gives the column names. - -In the `json-summary` file values for the maximum and average statistics -are given in JSON format. This file is rewritten every `interval` seconds -with the current summary values. Use the `--units` option to see exactly -which units are used for each metric (the value of `1` for a unit means -it is a pure number). - -In the `log-filename` output file, log messages (e.g., errors, warnings etc.) -are written. - -Monitoring of CPU, I/O and memory is reliably accurate, at least to within -the sampling time. Monitoring of network I/O is **not reliable** unless the -monitored process is isolated from other processes performing network I/O -(it gives an upper bound on the network activity, but the monitoring is -per network device as Linux does not give per-process network data by -default). - -### Visualisation - -The `prmon_plot.py` script (Python3) can be used to plot the outputs of prmon from the -timestamped output file (usually `prmon.txt`). Some examples include: - -* Memory usage as a function of wall-time: -```sh -prmon_plot.py --input prmon.txt --xvar wtime --yvar vmem,pss,rss,swap --yunit GB -``` -![](example-plots/PrMon_wtime_vs_vmem_pss_rss_swap.png) - -* Rate of change in memory usage as a function of wall-time: -```sh -prmon_plot.py --input prmon.txt --xvar wtime --yvar vmem,pss,rss,swap --diff --yunit MB -``` -![](example-plots/PrMon_wtime_vs_diff_vmem_pss_rss_swap.png) - -* Rate of change in CPU usage as a function of wall-time with stacked -user and system utilizations: -```sh -prmon_plot.py --input prmon.txt --xvar wtime --yvar utime,stime --yunit SEC --diff --stacked -``` -![](example-plots/PrMon_wtime_vs_diff_utime_stime.png) - -The plots above, as well as the input `prmon.txt` file that is used -to produce them, can be found under the `example-plots` folder. - -The script allows the user to specify variables, their units, plotting -style (stacked vs overlaid), as well as the format of the output image. -Use `-h` for more information. - - -### Data Compression - -The `prmon_compress_output.py` script (Python3) can be used to compress the output file -while keeping the most relevant information. - -The compression algorithm works as follows: -* For the number of processes, threads, and GPUs, only the measurements that are different with respect to the previous ones are kept. -* For all other metrics, only the measurements that satisfy an interpolation condition are kept. - -This latter condition can be summarized as: -* For any three neighboring (and time-ordered) measurements, A, B, and C, B is deleted if the linear interpolation between A and C is consistent with B ± *threshold*. Otherwise, it's retained. The *threshold* can be configured via the `--precision` parameter (default: 0.05, i.e. 5%) - - -The time index of the final output will be the union of the algorithm outputs of the single -time series. Each series will have NA values where a point was deleted at a kept index and, unless otherwise -specified by the `--skip-interpolate` parameter, will be linearly interpolated to maintain a consistent number of data points -and the result will be rounded to the nearest integer for consistency with the original input. - -If the `--skip-interpolate` parameter is passed, deleted values will be written as empty strings in the output file, and will be interpreted -as `NA` values when imported into Pandas. - -Example: -```sh -prmon_compress_output.py --input prmon.txt --precision 0.3 --skip-interpolate -``` - - -## Feedback and Contributions - -We're very happy to get feedback on prmon as well as suggestions for future -development. Please have a look at our [Contribution -Guide](doc/CONTRIBUTING.md). - - -### Profiling - -To build prmon with profiling, set one of the CMake variables -`PROFILE_GPROF` or `PROFILE_GPERFTOOLS` to `ON`. This enables -GNU prof profiling or gperftools profiling, respectively. -If your gperftools are in a non-standard place, pass a hint -to CMake using `Gperftools_ROOT_DIR`. - - -# Copyright - -Copyright (c) 2018-2025 CERN. - -## Contributors ✨ - -Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): - - - - - - - - - - - - - - - - - - - - - - - - - -
Graeme A Stewart
Graeme A Stewart

πŸš‡ ⚠️ πŸ’»
Alaettin Serhan Mete
Alaettin Serhan Mete

πŸš‡ ⚠️ πŸ’»
Anubhab Das
Anubhab Das

⚠️ πŸ’»
Chuck Atkins
Chuck Atkins

πŸš‡
Ismail Ryabchuk
Ismail Ryabchuk

πŸ’»
Chris Burr
Chris Burr

πŸ’»
Riccardo Maganza
Riccardo Maganza

πŸ’»
Miguel Gila
Miguel Gila

πŸ’»
Dan Protopopescu
Dan Protopopescu

πŸ’»
Juan Miguel Carceller
Juan Miguel Carceller

πŸš‡
Siddhant2306
Siddhant2306

πŸ“– πŸ’»
Eyüp Kerem Baş
Eyüp Kerem Baş

πŸ“– πŸ’»
utkarshasinha
utkarshasinha

πŸ“–
Aditya
Aditya

πŸ’»
- - - - - - -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! +# curlmon +On machines with prometheus exporters running, this collector is able to read their output via curl and store different power consumption metrics: DCMI, IPMI, RAPL and HWMON. These metrics together with $f_{usage}$ provides a precise estimation of the power consumption attributable to the monitored process. diff --git a/package/src/curlmon.cpp b/package/src/curlmon.cpp index 1fb9d78..a604117 100644 --- a/package/src/curlmon.cpp +++ b/package/src/curlmon.cpp @@ -23,8 +23,7 @@ static size_t curl_write_cb(char* ptr, size_t size, size_t nmemb, return size * nmemb; } - -// Constructor; uses RAII pattern to be valid after construction +// Initiate curl_stats curlmon::curlmon() : curl_stats{} { log_init(MONITOR_NAME); #undef MONITOR_NAME @@ -83,6 +82,7 @@ double curlmon::parse_metric_line(const std::string& line, void curlmon::update_stats(const std::vector& /*pids*/, const std::string /*read_path*/) { + // Collect matching lines from all URLs std::vector dcmi_lines{}; std::vector rapl_lines{}; @@ -108,6 +108,8 @@ void curlmon::update_stats(const std::vector& /*pids*/, ipmi_lines.push_back(line); if (lower.find("node_hwmon_power_watt") != std::string::npos) hwmon_lines.push_back(line); + else if (lower.find("node_hwmon_power_average_watt") != std::string::npos) + hwmon_lines.push_back(line); } } catch (...) { ++failed_urls; // Silencioso: no se loga nada por URL individual From 9e91dce8b49067444f7a4ae0d029b85feebb3aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Wed, 14 Oct 2026 15:51:46 +0200 Subject: [PATCH 05/14] Logic corrections, bool valid variable --- package/src/curlmon.cpp | 84 +++++++++++++++++++++++++++++------------ package/src/curlmon.h | 11 +++++- package/src/machmon.cpp | 22 +++++++---- package/src/machmon.h | 10 ++++- 4 files changed, 92 insertions(+), 35 deletions(-) diff --git a/package/src/curlmon.cpp b/package/src/curlmon.cpp index a604117..8f91fb5 100644 --- a/package/src/curlmon.cpp +++ b/package/src/curlmon.cpp @@ -23,13 +23,42 @@ static size_t curl_write_cb(char* ptr, size_t size, size_t nmemb, return size * nmemb; } -// Initiate curl_stats +// Constructor; uses RAII pattern to be valid after construction curlmon::curlmon() : curl_stats{} { log_init(MONITOR_NAME); #undef MONITOR_NAME for (const auto& param : params) { curl_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); } + valid = false; + + if (metrics_urls.empty()) { + debug("No exporter URLs provided, curlmon disabled"); + return; + } + + bool reachable = false; + std::vector dcmi, rapl, ipmi, hwmon; + for (const auto& url : metrics_urls) { + try { + collect_lines(fetch_metrics(url), dcmi, rapl, ipmi, hwmon); + reachable = true; + } catch (const std::exception& e) { + debug("curlmon: endpoint " + url + " not usable: " + e.what()); + } + } + + if (!reachable) { + info("Cannot connect to any exporter endpoint. curlmon disabled."); + return; + } + if (dcmi.empty() && rapl.empty() && ipmi.empty() && hwmon.empty()) { + info("Exporter reachable but none of the dcmi/rapl/ipmi/hwmon metrics " + "were found. curlmon disabled."); + return; + } + + valid = true; } // Fetch Prometheus text from the ctt URL @@ -41,7 +70,7 @@ std::string curlmon::fetch_metrics(const std::string& url) const { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2L); CURLcode res = curl_easy_perform(curl); curl_easy_cleanup(curl); @@ -80,6 +109,30 @@ double curlmon::parse_metric_line(const std::string& line, } +void curlmon::collect_lines(const std::string& body, + std::vector& dcmi, + std::vector& rapl, + std::vector& ipmi, + std::vector& hwmon) const { + std::istringstream stream{body}; + std::string line{}; + while (std::getline(stream, line)) { + std::string lower = line; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + if (lower.find("ipmi_dcmi_power_consumption_watts") != std::string::npos) + dcmi.push_back(line); + if (lower.find("node_rapl_package_joules_total") != std::string::npos) + rapl.push_back(line); + if (lower.find("ipmi_power_watts") != std::string::npos) + ipmi.push_back(line); + if (lower.find("node_hwmon_power_watt") != std::string::npos) + hwmon.push_back(line); + else if (lower.find("node_hwmon_power_average_watt") != std::string::npos) + hwmon.push_back(line); + } +} + + void curlmon::update_stats(const std::vector& /*pids*/, const std::string /*read_path*/) { @@ -89,36 +142,19 @@ void curlmon::update_stats(const std::vector& /*pids*/, std::vector ipmi_lines{}; std::vector hwmon_lines{}; - if (connection_failed) return; - int failed_urls = 0; for (const auto& url : metrics_urls) { try { - const std::string body = fetch_metrics(url); - std::istringstream stream{body}; - std::string line{}; - while (std::getline(stream, line)) { - std::string lower = line; - std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); - if (lower.find("ipmi_dcmi_power_consumption_watts") != std::string::npos) - dcmi_lines.push_back(line); - if (lower.find("node_rapl_package_joules_total") != std::string::npos) - rapl_lines.push_back(line); - if (lower.find("ipmi_power_watts") != std::string::npos) - ipmi_lines.push_back(line); - if (lower.find("node_hwmon_power_watt") != std::string::npos) - hwmon_lines.push_back(line); - else if (lower.find("node_hwmon_power_average_watt") != std::string::npos) - hwmon_lines.push_back(line); - } + collect_lines(fetch_metrics(url), dcmi_lines, rapl_lines, + ipmi_lines, hwmon_lines); } catch (...) { - ++failed_urls; // Silencioso: no se loga nada por URL individual + ++failed_urls; } } if (failed_urls == (int)metrics_urls.size()) { - error("Cannot connect to any exporter endpoint, disabling curlmon"); - connection_failed = true; + info("Cannot connect to any exporter endpoint, disabling curlmon"); + valid = false; return; } diff --git a/package/src/curlmon.h b/package/src/curlmon.h index b5260f7..05c2fa5 100644 --- a/package/src/curlmon.h +++ b/package/src/curlmon.h @@ -28,8 +28,15 @@ class curlmon final : public Imonitor, public MessageBase { {"rapl_1", "J", "RAPL package 1 energy in joules"}, {"rapl_2", "J", "RAPL package 2 energy in joules"} }; + + // Extract lines from the Prometheus text output for each metric type + void collect_lines(const std::string& body, + std::vector& dcmi, + std::vector& rapl, + std::vector& ipmi, + std::vector& hwmon) const; - bool connection_failed{false}; // To disable monitor if all ports fail + bool valid{false}; // To disable monitor if all ports fail or no metric is read prmon::monitored_list curl_stats; @@ -55,7 +62,7 @@ class curlmon final : public Imonitor, public MessageBase { prmon::parameter_list const get_parameter_list(); void const get_hardware_info(nlohmann::json& hw_json); void const get_unit_info(nlohmann::json& unit_json); - bool const is_valid() { return true; } + bool const is_valid() { return valid; } }; REGISTER_MONITOR(Imonitor, curlmon, "Monitor power consumption - prometheus") diff --git a/package/src/machmon.cpp b/package/src/machmon.cpp index b9e169c..f1dc16f 100644 --- a/package/src/machmon.cpp +++ b/package/src/machmon.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "utils.h" @@ -23,29 +24,37 @@ machmon::machmon() : mach_stats{} { for (const auto& param : params) { mach_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); } + + valid = true; } // Read the first line of /proc/stat and return {utime, stime, ntime} static std::tuple read_proc_stat( - const std::string& read_path) { - std::string stat_path = "/proc/stat"; + const std::string& /*read_path*/) { + const std::string stat_path = "/proc/stat"; std::ifstream proc_stat{stat_path}; - if (!proc_stat) + if (!proc_stat) { throw std::runtime_error("Cannot open " + stat_path); + } std::string cpu_label{}; long user{}, nice{}, system{}; // First line: "cpu user nice system idle ..." - if (!(proc_stat >> cpu_label >> user >> nice >> system)) + if (!(proc_stat >> cpu_label >> user >> nice >> system)) { throw std::runtime_error("Unexpected format in " + stat_path); + } - return {user, system, nice}; + return std::make_tuple(user, system, nice); } void machmon::update_stats(const std::vector& /*pids*/, const std::string read_path) { + if (!valid) { + return; + } + long utime, stime, ntime; std::tie(utime, stime, ntime) = read_proc_stat(read_path); @@ -61,7 +70,6 @@ void machmon::update_stats(const std::vector& /*pids*/, mach_stats.at("utime_total").set_value((utime - initial_utime) / clk); mach_stats.at("stime_total").set_value((stime - initial_stime) / clk); mach_stats.at("ntime_total").set_value((ntime - initial_ntime) / clk); - } // Return the summed counters @@ -88,7 +96,7 @@ prmon::monitored_average_map const machmon::get_json_average_stats( // Return the parameter list prmon::parameter_list const machmon::get_parameter_list() { return params; } -// No extra hardware info needed here +// No extra hardware information needed here void const machmon::get_hardware_info(nlohmann::json& /*hw_json*/) { return; } diff --git a/package/src/machmon.h b/package/src/machmon.h index 387d241..c949fe9 100644 --- a/package/src/machmon.h +++ b/package/src/machmon.h @@ -21,18 +21,21 @@ class machmon final : public Imonitor, public MessageBase { private: + // Setup the parameters to monitor here const prmon::parameter_list params = { {"utime_total", "s", "Total user CPU time from /proc/stat"}, {"stime_total", "s", "Total system CPU time from /proc/stat"}, {"ntime_total", "s", "Total nice CPU time from /proc/stat"}, }; - prmon::monitored_list mach_stats; // era monitored_value_map + // Dynamic monitoring container + prmon::monitored_list mach_stats; long initial_utime{0L}; long initial_stime{0L}; long initial_ntime{0L}; bool initialised{false}; + bool valid{false}; // To disable monitor if /proc/stat is not readable public: machmon(); @@ -40,14 +43,17 @@ class machmon final : public Imonitor, public MessageBase { void update_stats(const std::vector& pids, const std::string read_path = ""); + // These are the stat getter methods which retrieve current statistics prmon::monitored_value_map const get_text_stats(); prmon::monitored_value_map const get_json_total_stats(); prmon::monitored_average_map const get_json_average_stats( unsigned long long elapsed_clock_ticks); prmon::parameter_list const get_parameter_list(); + + // Hardware information getter to run once void const get_hardware_info(nlohmann::json& hw_json); void const get_unit_info(nlohmann::json& unit_json); - bool const is_valid() { return true; } + bool const is_valid() { return valid; } }; REGISTER_MONITOR(Imonitor, machmon, "Monitor full machine cpu time") From dcd783c4ad9efa8cdcbe3c6d480f933aabe1aac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Wed, 14 Oct 2026 16:03:49 +0200 Subject: [PATCH 06/14] Restore README --- README.md | 326 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 320 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bca3a05..7c5bc28 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,325 @@ [![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-) -This fork of prmon aims to test two additional collectors (curlmon and machmon) that can add information regarding power consumption. The idea stems from a proposal to improve job power consumption monitoring via an estimation of the usage of the machine by a given job. +[![Build Status][build-img]][build-link] [![License][license-img]][license-url] [![Codefactor][codefactor-img]][codefactor-url] [![HSF Project][hsf-project-img]][hsf-project-url] -# machmon -Similarly to cpumon, this collector reads the cpu times of the whole machine (utime, stime and ntime). By comparing pidtimes with machtimes, we can compute a ratio that estimates the fraction of the machine that is being used by the monitored process. -$f_{usage} = \frac{t_{PID}}{t_{total}}$ +[build-img]: https://github.com/hsf/prmon/actions/workflows/ci.yml/badge.svg?branch=main +[build-link]: https://github.com/hsf/prmon/actions/workflows/ci.yml?query=branch%3Amain +[license-img]: https://img.shields.io/github/license/hsf/prmon.svg +[license-url]: https://github.com/hsf/prmon/blob/main/LICENSE +[codefactor-img]: https://www.codefactor.io/repository/github/HSF/prmon/badge +[codefactor-url]: https://www.codefactor.io/repository/github/HSF/prmon +[hsf-project-img]: https://raw.githubusercontent.com/HSF/hsf.github.io/refs/heads/main/images/HSF-logo/HSF-Affiliated.svg +[hsf-project-url]: https://hepsoftwarefoundation.org/projects/affiliated.html -# curlmon -On machines with prometheus exporters running, this collector is able to read their output via curl and store different power consumption metrics: DCMI, IPMI, RAPL and HWMON. These metrics together with $f_{usage}$ provides a precise estimation of the power consumption attributable to the monitored process. +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2554202.svg)](https://doi.org/10.5281/zenodo.2554202) + +The PRocess MONitor is a small stand alone program that can monitor +the resource consumption of a process and its children. This is +useful in the context of the WLCG/HSF working group to evaluate +the costs and performance of HEP workflows in WLCG. In a previous +incarnation (MemoryMonitor) it has been used by ATLAS for some time to +gather data on resource consumption by production jobs. One of its +most useful features is to use smaps to correctly calculate the +*Proportional Set Size* in the group of processes monitored, which +is a much better indication of the true memory consumption of +a group of processes where children share many pages. + +prmon currently runs on Linux machines as it requires access to the +`/proc` interface to process statistics. + +## Build and Deployment + +### Cloning the project + +As prmon has dependencies on submodules, clone the project as + + git clone --recurse-submodules https://github.com/HSF/prmon.git + +### Building the project + +Building prmon requires a C++ compiler that fully supports C++11, +and CMake version 3.10 or higher. It also has dependencies on: + + - [Niels Lohmann JSON libraries](https://github.com/nlohmann/json) + - `nlohmann-json-dev` in Ubuntu 18, `nlohmann-json3-dev` in Ubuntu 20 + - [spdlog: Fast C++ logging library](https://github.com/gabime/spdlog) + - `libspdlog-dev` in Ubuntu + +and can use either external system-supplied versions or internal copies +provided by submodules. + +Building is usually as simple as: + + mkdir build + cd build + cmake -DCMAKE_INSTALL_PREFIX= -S .. -B . + make -j + make install + +Unless otherwise specified, the default behavior for dependencies is to first +try to find an external version and fall back to the internal submodule copy +if not found. To explicitly force the use of either add any of the following +configure options: + - `-DUSE_EXTERNAL_NLOHMANN_JSON={TRUE,FALSE,`**`AUTO`**`}` + - `ON`, `TRUE`: Force an external version and fail if not found. + - `OFF`, `FALSE`: Require the internal copy be used. + - **`AUTO`**: Search for an external version and fall back to the + internal copy if not found. + - `-Dnlohmann_json_DIR=/path/to/config` + - The path to the directory containing `nlohmann_jsonConfig.cmake`. + Necessary if nlohmann_json is not installed into CMake's search path. + - `-DUSE_EXTERNAL_SPDLOG={TRUE,FALSE,`**`AUTO`**`}` + - `ON`, `TRUE`: Force an external version and fail if not found. + - `OFF`, `FALSE`: Require the internal copy be used. + - **`AUTO`**: Search for an external version and fall back to the + internal copy if not found. + - `-Dspdlog_DIR=/path/to/config` + - The path to the directory containing `spdlogConfig.cmake`. + Necessary if spdlog is not installed into CMake's search path. + +The option `-DCMAKE_BUILD_TYPE` can switch between all of the standard +build types. The default is `Release`; use `RelWithDebInfo` if you want +debug symbols. + +To build a statically linked version of `prmon`, set the `BUILD_STATIC` +CMake variable to `ON` (e.g., adding `-DBUILD_STATIC=ON` to the +command line). + +Note that in a build environment with CVMFS available the C++ compiler +and CMake can be taken by setting up a recent LCG release. + +To enable pulling and building the gtest framework as well as tests dependent on gtest, +build with `-DBUILD_GTESTS=ON`. + +### Creating a package with CPack + +A cpack based package can be created by invoking + + make package + +### Running the tests + +To run the tests of the project, first build it and then invoke + + make test + +Running the tests requires Python version 3.6 or higher. + +## Running + +The `prmon` binary is invoked with the following arguments: + +```sh +prmon [--pid PPP] [--filename prmon.txt] [--json-summary prmon.json] \ + [--log-filename prmon.log] [--interval 30] \ + [--suppress-hw-info] [--units] [--netdev DEV] \ + [--disable MON1] [--level LEV] [--level MON:LEV] \ + [--fast-memmon] \ + [-- prog arg arg ...] +``` + +* `--pid` the 'mother' PID to monitor (all children in the same process tree are monitored as well) +* `--filename` output file for time-stamped monitored values +* `--json-summary` output file for summary data written in JSON format +* `--log-filename` output file for log messages +* `--interval` time, in seconds, between monitoring snapshots +* `--suppress-hw-info` flag that turns-off hardware information collection +* `--units` add information on units for each metric to JSON file +* `--netdev` restricts network statistics to one (or more) network devices +* `--disable` is used to disable specific monitors (and can be specified multiple times); + the default is that `prmon` monitors everything that it can + * Note that the `wallmon` monitor is the only monitor that cannot be disabled +* `--level` is used to set the logging level for monitors + * `--level LEV` sets the level for all monitors to LEV + * `--level MON:LEV` sets the level for monitor MON to LEV + * The valid levels are `trace`, `debug`, `info`, `warn`, `error`, `critical` +* `--fast-memmon` toggles on fast memory monitoring using `smaps_rollup` +* `--` after this argument the following arguments are treated as a program to invoke + and remaining arguments are passed to it; `prmon` will then monitor this process + instead of being given a PID via `--pid` + +`prmon` will exit with `1` if there is a problem with inconsistent or +incomplete arguments. If `prmon` starts a program itself (using `--`) then +`prmon` will exit with the exit code of the child process. + +When invoked with `-h` or `--help` usage information is printed, as well as a +list of all available monitoring components. + +### Fast Memory Monitoring + +When invoked with `--fast-memmon` `prmon` uses the `smaps_rollup` files +that contain pre-summed memory information for each monitored process. +This is a faster approach compared to the default behavior, +where `prmon` aggregates the results itself by going over each of the monitored +processes' mappings one by one. + +If the current kernel doesn't support `smaps_rollup` then the default +approach is used. Users should also note that fast memory monitoring +might not contain all metrics that the default approach supports, e.g., +`vmem`. In that case, the missing metric will be omitted in the output. +If any of these issues are encountered, a relevant message is printed +to notify the user. + +### NVIDIA GPU Monitoring + +When NVIDIA GPUs are present, `prmon` will preferentially use the +NVIDIA Management Library (NVML) for direct GPU monitoring via +`libnvidia-ml.so`. If NVML is not available or fails to initialise, +`prmon` falls back to parsing the output of `nvidia-smi`. +No extra build-time dependencies are needed: NVML symbols are loaded +at runtime via `dlopen`, so `prmon` can be compiled on machines without +NVIDIA drivers and will detect GPU support when run on a machine that +has them. + +### Environment Variables + +The `PRMON_DISABLE_MONITOR` environment variable can be used to specify a comma +separated list of monitor names that will be disabled. This is useful when +`prmon` is being invoked by some other part of a job or workflow, so the user +does not have direct access to the command line options used. e.g. + +```sh +export PRMON_DISABLE_MONITOR=nvidiamon +other_code_that_invokes_prmon +... +``` + +Disables the `nvidiamon` monitor. + +## Outputs + +In the `filename` output file, plain text with statistics written every +`interval` seconds are written. The first line gives the column names. + +In the `json-summary` file values for the maximum and average statistics +are given in JSON format. This file is rewritten every `interval` seconds +with the current summary values. Use the `--units` option to see exactly +which units are used for each metric (the value of `1` for a unit means +it is a pure number). + +In the `log-filename` output file, log messages (e.g., errors, warnings etc.) +are written. + +Monitoring of CPU, I/O and memory is reliably accurate, at least to within +the sampling time. Monitoring of network I/O is **not reliable** unless the +monitored process is isolated from other processes performing network I/O +(it gives an upper bound on the network activity, but the monitoring is +per network device as Linux does not give per-process network data by +default). + +### Visualisation + +The `prmon_plot.py` script (Python3) can be used to plot the outputs of prmon from the +timestamped output file (usually `prmon.txt`). Some examples include: + +* Memory usage as a function of wall-time: +```sh +prmon_plot.py --input prmon.txt --xvar wtime --yvar vmem,pss,rss,swap --yunit GB +``` +![](example-plots/PrMon_wtime_vs_vmem_pss_rss_swap.png) + +* Rate of change in memory usage as a function of wall-time: +```sh +prmon_plot.py --input prmon.txt --xvar wtime --yvar vmem,pss,rss,swap --diff --yunit MB +``` +![](example-plots/PrMon_wtime_vs_diff_vmem_pss_rss_swap.png) + +* Rate of change in CPU usage as a function of wall-time with stacked +user and system utilizations: +```sh +prmon_plot.py --input prmon.txt --xvar wtime --yvar utime,stime --yunit SEC --diff --stacked +``` +![](example-plots/PrMon_wtime_vs_diff_utime_stime.png) + +The plots above, as well as the input `prmon.txt` file that is used +to produce them, can be found under the `example-plots` folder. + +The script allows the user to specify variables, their units, plotting +style (stacked vs overlaid), as well as the format of the output image. +Use `-h` for more information. + + +### Data Compression + +The `prmon_compress_output.py` script (Python3) can be used to compress the output file +while keeping the most relevant information. + +The compression algorithm works as follows: +* For the number of processes, threads, and GPUs, only the measurements that are different with respect to the previous ones are kept. +* For all other metrics, only the measurements that satisfy an interpolation condition are kept. + +This latter condition can be summarized as: +* For any three neighboring (and time-ordered) measurements, A, B, and C, B is deleted if the linear interpolation between A and C is consistent with B ± *threshold*. Otherwise, it's retained. The *threshold* can be configured via the `--precision` parameter (default: 0.05, i.e. 5%) + + +The time index of the final output will be the union of the algorithm outputs of the single +time series. Each series will have NA values where a point was deleted at a kept index and, unless otherwise +specified by the `--skip-interpolate` parameter, will be linearly interpolated to maintain a consistent number of data points +and the result will be rounded to the nearest integer for consistency with the original input. + +If the `--skip-interpolate` parameter is passed, deleted values will be written as empty strings in the output file, and will be interpreted +as `NA` values when imported into Pandas. + +Example: +```sh +prmon_compress_output.py --input prmon.txt --precision 0.3 --skip-interpolate +``` + + +## Feedback and Contributions + +We're very happy to get feedback on prmon as well as suggestions for future +development. Please have a look at our [Contribution +Guide](doc/CONTRIBUTING.md). + + +### Profiling + +To build prmon with profiling, set one of the CMake variables +`PROFILE_GPROF` or `PROFILE_GPERFTOOLS` to `ON`. This enables +GNU prof profiling or gperftools profiling, respectively. +If your gperftools are in a non-standard place, pass a hint +to CMake using `Gperftools_ROOT_DIR`. + + +# Copyright + +Copyright (c) 2018-2025 CERN. + +## Contributors ✨ + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + + + + + + + + + + + + + + + + + + + + + + + +
Graeme A Stewart
Graeme A Stewart

πŸš‡ ⚠️ πŸ’»
Alaettin Serhan Mete
Alaettin Serhan Mete

πŸš‡ ⚠️ πŸ’»
Anubhab Das
Anubhab Das

⚠️ πŸ’»
Chuck Atkins
Chuck Atkins

πŸš‡
Ismail Ryabchuk
Ismail Ryabchuk

πŸ’»
Chris Burr
Chris Burr

πŸ’»
Riccardo Maganza
Riccardo Maganza

πŸ’»
Miguel Gila
Miguel Gila

πŸ’»
Dan Protopopescu
Dan Protopopescu

πŸ’»
Juan Miguel Carceller
Juan Miguel Carceller

πŸš‡
Siddhant2306
Siddhant2306

πŸ“– πŸ’»
Eyüp Kerem Baş
Eyüp Kerem Baş

πŸ“– πŸ’»
utkarshasinha
utkarshasinha

πŸ“–
Aditya
Aditya

πŸ’»
+ + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! From 684c46d5f3918ea489035f451c89ebf4249271d3 Mon Sep 17 00:00:00 2001 From: Graeme Andrew Stewart Date: Fri, 16 Oct 2026 14:06:11 +0200 Subject: [PATCH 07/14] Add libcurl-devel for CI Power monitoring requires curl to retrieve information, which requires libcurl-devel on almalinux 10 --- .github/scripts/build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/build-test.sh b/.github/scripts/build-test.sh index beb6ccc..885a8b0 100755 --- a/.github/scripts/build-test.sh +++ b/.github/scripts/build-test.sh @@ -12,7 +12,7 @@ echo "Starting build and test for platform $PLATFORM, compiler suite $COMPILER" # In vanilla containers we need to install the necessary development packages if [[ "$PLATFORM" == almalinux* ]]; then echo "Installing additional RPM development packages" - dnf install -y gcc-c++ cmake boost boost-devel make clang + dnf install -y gcc-c++ cmake boost boost-devel make clang libcurl-devel fi if [[ $PLATFORM == ubuntu* ]]; then From f8081c4f0e452c447b73f09aee9281f5fa172ea6 Mon Sep 17 00:00:00 2001 From: Graeme Andrew Stewart Date: Fri, 16 Oct 2026 14:15:36 +0200 Subject: [PATCH 08/14] Further CI curl development fixes --- .github/scripts/build-test.sh | 2 +- .github/scripts/flake8.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/build-test.sh b/.github/scripts/build-test.sh index 885a8b0..c975a82 100755 --- a/.github/scripts/build-test.sh +++ b/.github/scripts/build-test.sh @@ -18,7 +18,7 @@ fi if [[ $PLATFORM == ubuntu* ]]; then echo "Installing additional DEB development packages" apt update - apt -y install g++ clang libboost-dev cmake make + apt -y install g++ clang libboost-dev cmake make libcurl4-openssl-dev fi # If environment variables CXX or CMAKE are defined then use these diff --git a/.github/scripts/flake8.sh b/.github/scripts/flake8.sh index 200cd85..4b9c829 100755 --- a/.github/scripts/flake8.sh +++ b/.github/scripts/flake8.sh @@ -3,7 +3,7 @@ # First we need to install Python and Flake 8 # N.B. this is for an Alma Linux container (specifically almalinux 9 # but should be quite generic) -dnf -y install python3 python-pip cmake gcc-c++ boost boost-devel make +dnf -y install python3 python-pip cmake gcc-c++ boost boost-devel make libcurl-devel pip install flake8 # Configure and run flake8 target From 7d0012a4e2f57ca45ed187e7231a2f0b6f9eb5e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Fri, 16 Oct 2026 15:04:18 +0200 Subject: [PATCH 09/14] Removed average values for machmon and redudndant blank line at curlmon. --- package/src/curlmon.cpp | 1 - package/src/machmon.h | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/package/src/curlmon.cpp b/package/src/curlmon.cpp index 8f91fb5..d39cee7 100644 --- a/package/src/curlmon.cpp +++ b/package/src/curlmon.cpp @@ -135,7 +135,6 @@ void curlmon::collect_lines(const std::string& body, void curlmon::update_stats(const std::vector& /*pids*/, const std::string /*read_path*/) { - // Collect matching lines from all URLs std::vector dcmi_lines{}; std::vector rapl_lines{}; diff --git a/package/src/machmon.h b/package/src/machmon.h index c949fe9..bcff55e 100644 --- a/package/src/machmon.h +++ b/package/src/machmon.h @@ -23,9 +23,9 @@ class machmon final : public Imonitor, public MessageBase { private: // Setup the parameters to monitor here const prmon::parameter_list params = { - {"utime_total", "s", "Total user CPU time from /proc/stat"}, - {"stime_total", "s", "Total system CPU time from /proc/stat"}, - {"ntime_total", "s", "Total nice CPU time from /proc/stat"}, + {"utime_total", "s", ""}, + {"stime_total", "s", ""}, + {"ntime_total", "s", ""}, }; // Dynamic monitoring container From 15c402df13e929be501d9d742a1394cbb5acf2a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Mon, 26 Oct 2026 12:26:35 +0100 Subject: [PATCH 10/14] Rename collector, several reviewed corrections --- package/CMakeLists.txt | 2 +- package/src/machmon.h | 3 - package/src/{curlmon.cpp => powermon.cpp} | 180 ++++++++++++++++------ package/src/{curlmon.h => powermon.h} | 27 ++-- 4 files changed, 145 insertions(+), 67 deletions(-) rename package/src/{curlmon.cpp => powermon.cpp} (51%) rename package/src/{curlmon.h => powermon.h} (78%) diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt index 1c9eeb8..2c9fa6c 100644 --- a/package/CMakeLists.txt +++ b/package/CMakeLists.txt @@ -11,7 +11,7 @@ add_executable(prmon src/prmon.cpp src/wallmon.cpp src/memmon.cpp src/machmon.cpp - src/curlmon.cpp + src/powermon.cpp ) target_link_libraries(prmon PRIVATE diff --git a/package/src/machmon.h b/package/src/machmon.h index bcff55e..662cc0d 100644 --- a/package/src/machmon.h +++ b/package/src/machmon.h @@ -2,9 +2,6 @@ // License Apache2 - see LICENCE file // Full CPU monitoring class -// - -#pragma once #ifndef PRMON_MACHMON_H #define PRMON_MACHMON_H 1 diff --git a/package/src/curlmon.cpp b/package/src/powermon.cpp similarity index 51% rename from package/src/curlmon.cpp rename to package/src/powermon.cpp index d39cee7..9eca30b 100644 --- a/package/src/curlmon.cpp +++ b/package/src/powermon.cpp @@ -1,7 +1,7 @@ // Copyright (C) 2018-2025 CERN // License Apache2 - see LICENCE file -#include "curlmon.h" +#include "powermon.h" #include @@ -11,79 +11,150 @@ #include #include +#include +#include +#include +#include + + #include "utils.h" -#define MONITOR_NAME "curlmon" +#define MONITOR_NAME "powermon" + +// Maximum response size (8 MB) +static const curl_off_t kMaxResponseSize = 8 * 1024 * 1024; // libcurl write, appending data to a std::string static size_t curl_write_cb(char* ptr, size_t size, size_t nmemb, void* userdata) { auto* buf = reinterpret_cast(userdata); - buf->append(ptr, size * nmemb); - return size * nmemb; + const size_t new_size = size * nmemb; + + // Return 0 if the new size would exceed the maximum allowed response size + if (buf->size() + new_size > static_cast(kMaxResponseSize)) return 0; + + buf->append(ptr, new_size); + return new_size; } // Constructor; uses RAII pattern to be valid after construction -curlmon::curlmon() : curl_stats{} { +powermon::powermon() : power_stats{} { log_init(MONITOR_NAME); #undef MONITOR_NAME + +// Global init for curl, explicitly + if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) { + error("Failed to initialise libcurl"); + return; + } + for (const auto& param : params) { - curl_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); + if (param.get_name() == "rapl_1" || param.get_name() == "rapl_2") { + + // RAPL values are monotonic, so we set the flag to true + power_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); + } else { + power_stats.emplace(param.get_name(), prmon::monitored_value(param, false)); + } } valid = false; + + // URLs from the environment variable PROMETHEUS_LOCAL_URL, if it exists + // Expected format: "http://localhost:9100/metrics,http://localhost:9290/metrics" + const char* env_string = std::getenv("PROMETHEUS_LOCAL_URL"); + + if (env_string) { + unsigned pos{0}, start{0}; + while (true) { + if (env_string[pos] == ',' || env_string[pos] == '\0') { + if (pos > start) + metrics_urls.push_back(std::string(env_string + start, pos - start)); + if (env_string[pos] == '\0') break; + start = ++pos; + } else { + ++pos; + } + } + } + + // Use default values if no url was provided in the environment variable + if (metrics_urls.empty()) metrics_urls = default_metrics_urls; if (metrics_urls.empty()) { - debug("No exporter URLs provided, curlmon disabled"); + debug("No exporter URLs provided, powermon disabled"); return; } bool reachable = false; + std::vector dcmi, rapl, ipmi, hwmon; for (const auto& url : metrics_urls) { try { collect_lines(fetch_metrics(url), dcmi, rapl, ipmi, hwmon); reachable = true; } catch (const std::exception& e) { - debug("curlmon: endpoint " + url + " not usable: " + e.what()); + debug("powermon: endpoint " + url + " not usable: " + e.what()); } } if (!reachable) { - info("Cannot connect to any exporter endpoint. curlmon disabled."); + info("Cannot connect to any exporter endpoint. powermon disabled."); return; } if (dcmi.empty() && rapl.empty() && ipmi.empty() && hwmon.empty()) { info("Exporter reachable but none of the dcmi/rapl/ipmi/hwmon metrics " - "were found. curlmon disabled."); + "were found. powermon disabled."); return; } valid = true; } +// Destructor: closes libcurl global state +powermon::~powermon() { + curl_global_cleanup(); +} + // Fetch Prometheus text from the ctt URL -std::string curlmon::fetch_metrics(const std::string& url) const { +std::string powermon::fetch_metrics(const std::string& url) { std::string response{}; CURL* curl = curl_easy_init(); - if (!curl) throw std::runtime_error("Failed to initialise libcurl"); - - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2L); - - CURLcode res = curl_easy_perform(curl); - curl_easy_cleanup(curl); + if (!curl) { + error("Failed to initialise libcurl, curl_easy_init() returned NULL"); + return response; + } - if (res != CURLE_OK) - throw std::runtime_error(std::string("curl request failed: ") + - curl_easy_strerror(res)); - return response; -} + // Define a function to check if any response is != 0 + CURLcode rc = CURLE_OK; + auto set_ok = [&](CURLoption option, auto value) { + rc = curl_easy_setopt(curl, option, value); + if (rc != CURLE_OK) { + error("curl_easy_setopt failed: " + std::string(curl_easy_strerror(rc))); + curl_easy_cleanup(curl); + valid = false; + return false; + }; + return true; + }; + + if (!set_ok(CURLOPT_URL, url.c_str()) || + !set_ok(CURLOPT_WRITEFUNCTION, curl_write_cb) || + !set_ok(CURLOPT_WRITEDATA, &response) || + !set_ok(CURLOPT_MAXFILESIZE_LARGE, kMaxResponseSize) || + !set_ok(CURLOPT_PROTOCOLS_STR, "http") || + !set_ok(CURLOPT_CONNECTTIMEOUT, 1L) || + !set_ok(CURLOPT_TIMEOUT, 2L) || + !set_ok(CURLOPT_PROXY, "")) { + CURLcode res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + return response; + }; + return {}; +}; // Parse a single Prometheus line: "metric_name{labels} value timestamp" // Returns the value if the line starts with metric_name, otherwise -1.0 -double curlmon::parse_metric_line(const std::string& line, +double powermon::parse_metric_line(const std::string& line, const std::string& metric_name) const { if (line.empty() || line[0] == '#') return -1.0; @@ -98,18 +169,22 @@ double curlmon::parse_metric_line(const std::string& line, else value_start += 1; - while (value_start < line.size() && std::isspace(line[value_start])) + while (value_start < line.size() && std::isspace(static_cast(line[value_start]))) ++value_start; try { - return std::stod(line.substr(value_start)); + const double val = std::stod(line.substr(value_start)); + if (std::isnan(val) || std::isinf(val)) return -1.0; // Return 1 if val is NaN or Inf + if (val < 0.0) return -1.0; + if (val >= static_cast(std::numeric_limits::max())) return -1.0; + return val; } catch (...) { return -1.0; } } -void curlmon::collect_lines(const std::string& body, +void powermon::collect_lines(const std::string& body, std::vector& dcmi, std::vector& rapl, std::vector& ipmi, @@ -118,7 +193,8 @@ void curlmon::collect_lines(const std::string& body, std::string line{}; while (std::getline(stream, line)) { std::string lower = line; - std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); if (lower.find("ipmi_dcmi_power_consumption_watts") != std::string::npos) dcmi.push_back(line); if (lower.find("node_rapl_package_joules_total") != std::string::npos) @@ -133,7 +209,7 @@ void curlmon::collect_lines(const std::string& body, } -void curlmon::update_stats(const std::vector& /*pids*/, +void powermon::update_stats(const std::vector& /*pids*/, const std::string /*read_path*/) { // Collect matching lines from all URLs std::vector dcmi_lines{}; @@ -152,7 +228,7 @@ void curlmon::update_stats(const std::vector& /*pids*/, } if (failed_urls == (int)metrics_urls.size()) { - info("Cannot connect to any exporter endpoint, disabling curlmon"); + error("Cannot connect to any exporter endpoint, disabling powermon"); valid = false; return; } @@ -200,17 +276,21 @@ void curlmon::update_stats(const std::vector& /*pids*/, } } catch (...) { hwmon = 0.0; } + // store function to handle value range checking and storing in power_stats + auto store = [&](const char* key, double value) { + if (!(value >= 0.0 && value < static_cast(std::numeric_limits::max()))) { + error(std::string("Value for ") + key + " is out of range, ignoring"); + value = 0.0; + } + power_stats.at(key).set_value(static_cast(value)); + }; + try { - curl_stats.at("dcmi").set_value( - static_cast(dcmi)); - curl_stats.at("rapl_1").set_value( - static_cast(rapl_1)); - curl_stats.at("rapl_2").set_value( - static_cast(rapl_2)); - curl_stats.at("ipmi").set_value( - static_cast(ipmi)); - curl_stats.at("hwmon").set_value( - static_cast(hwmon)); + store("dcmi", dcmi); + store("rapl_1", rapl_1); + store("rapl_2", rapl_2); + store("ipmi", ipmi); + store("hwmon", hwmon); } catch (const std::exception& e) { error(std::string("Error storing metrics: ") + e.what()); } @@ -218,29 +298,29 @@ void curlmon::update_stats(const std::vector& /*pids*/, // Return the monitored values -prmon::monitored_value_map const curlmon::get_text_stats() { +prmon::monitored_value_map const powermon::get_text_stats() { prmon::monitored_value_map stat_map{}; - for (const auto& value : curl_stats) { + for (const auto& value : power_stats) { stat_map[value.first] = value.second.get_value(); } return stat_map; } -prmon::monitored_value_map const curlmon::get_json_total_stats() { - return curlmon::get_text_stats(); +prmon::monitored_value_map const powermon::get_json_total_stats() { + return powermon::get_text_stats(); } -prmon::monitored_average_map const curlmon::get_json_average_stats( +prmon::monitored_average_map const powermon::get_json_average_stats( unsigned long long /*elapsed_clock_ticks*/) { static const prmon::monitored_average_map empty{}; return empty; } -prmon::parameter_list const curlmon::get_parameter_list() { return params; } +prmon::parameter_list const powermon::get_parameter_list() { return params; } -void const curlmon::get_hardware_info(nlohmann::json& /*hw_json*/) { return; } +void const powermon::get_hardware_info(nlohmann::json& /*hw_json*/) { return; } -void const curlmon::get_unit_info(nlohmann::json& unit_json) { +void const powermon::get_unit_info(nlohmann::json& unit_json) { prmon::fill_units(unit_json, params); return; } \ No newline at end of file diff --git a/package/src/curlmon.h b/package/src/powermon.h similarity index 78% rename from package/src/curlmon.h rename to package/src/powermon.h index 05c2fa5..971e2b5 100644 --- a/package/src/curlmon.h +++ b/package/src/powermon.h @@ -1,13 +1,10 @@ // Copyright (C) 2018-2025 CERN // License Apache2 - see LICENCE file -// Curl to prometheus monitoring class -// +// Power monitoring class -#pragma once - -#ifndef PRMON_CURLMON_H -#define PRMON_CURLMON_H 1 +#ifndef PRMON_POWERMON_H +#define PRMON_POWERMON_H 1 #include #include @@ -18,8 +15,9 @@ #include "MessageBase.h" #include "parameter.h" #include "registry.h" +#include -class curlmon final : public Imonitor, public MessageBase { +class powermon final : public Imonitor, public MessageBase { private: const prmon::parameter_list params = { {"dcmi", "W", "DCMI power consumption"}, @@ -38,19 +36,22 @@ class curlmon final : public Imonitor, public MessageBase { bool valid{false}; // To disable monitor if all ports fail or no metric is read - prmon::monitored_list curl_stats; + prmon::monitored_list power_stats; - const std::vector metrics_urls{ + const std::vector default_metrics_urls{ "http://localhost:9100/metrics", "http://localhost:9290/metrics" }; - std::string fetch_metrics(const std::string& url) const; + std::vector metrics_urls; + + std::string fetch_metrics(const std::string& url); double parse_metric_line(const std::string& line, const std::string& metric_name) const; public: - curlmon(); + powermon(); + ~powermon(); void update_stats(const std::vector& pids, const std::string read_path = ""); @@ -64,7 +65,7 @@ class curlmon final : public Imonitor, public MessageBase { void const get_unit_info(nlohmann::json& unit_json); bool const is_valid() { return valid; } }; -REGISTER_MONITOR(Imonitor, curlmon, "Monitor power consumption - prometheus") +REGISTER_MONITOR(Imonitor, powermon, "Monitor power consumption - prometheus") -#endif // PRMON_CURLMON_H \ No newline at end of file +#endif // PRMON_POWERMON_H From ec21da28884a6ee0ac86f9fcf78243938a8fc99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Thu, 29 Oct 2026 16:11:12 +0100 Subject: [PATCH 11/14] Updated to C++14, reorganized functions and comments for clarity and consistency, new precooked_tests --- CMakeLists.txt | 2 +- package/scripts/precook_test.py | 50 ++++ .../precooked_tests/drop/1/power/ipmi.prom | 13 ++ .../scripts/precooked_tests/drop/1/proc/stat | 1 + .../precooked_tests/drop/2/power/ipmi.prom | 13 ++ .../scripts/precooked_tests/drop/2/proc/stat | 1 + .../precooked_tests/drop/3/power/ipmi.prom | 13 ++ .../scripts/precooked_tests/drop/3/proc/stat | 1 + package/src/machmon.cpp | 11 +- package/src/powermon.cpp | 214 ++++++++++++------ package/src/powermon.h | 36 ++- 11 files changed, 274 insertions(+), 81 deletions(-) create mode 100644 package/scripts/precooked_tests/drop/1/power/ipmi.prom create mode 100644 package/scripts/precooked_tests/drop/1/proc/stat create mode 100644 package/scripts/precooked_tests/drop/2/power/ipmi.prom create mode 100644 package/scripts/precooked_tests/drop/2/proc/stat create mode 100644 package/scripts/precooked_tests/drop/3/power/ipmi.prom create mode 100644 package/scripts/precooked_tests/drop/3/proc/stat diff --git a/CMakeLists.txt b/CMakeLists.txt index 6311863..4eb3f82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,7 @@ set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" # Both of these issues can be resolved using CMake's compile features, see # # - https://cmake.org/cmake/help/v3.3/manual/cmake-compile-features.7.html -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index a3c253f..032053a 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -120,6 +120,44 @@ def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, print(param, file=f, end=" ") print("", file=f) +def make_machstat(proc_mach, fixed_value, rand=False): + mach_fname = os.path.join(proc_mach, "stat") + lim = 2**64 - 1 + if rand: + user, nice, system = random.randint(1,lim) + else: + user, nice, system = fixed_value, fixed_value, fixed_value + with open(mach_fname, "w") as f: + print("cpu", file = f, end = " ") # Relevant line + # 1 utime - Time in user mode + # 2 ntime - Time in nice mode + # 3 stime - Time in system mode + print(f"{user} {nice} {system}", file = f, end = " ") + + # 4 - 10 are never used, set to 0 + print("0 " * 7, file = f, end = "") + +def make_power(power_dir, fixed_value, rand = False): + power_fname = os.path.join(power_dir, "ipmi.prom") + if rand: + dcmi, ipmi, hwmon = (random.randint(0, 1000) for _ in range(3)) + rapl_0, rapl_1 = (random.randint(0, 1000000) for _ in range(3)) + else: + dcmi, ipmi, hwmon, rapl_0, rapl_1 = fixed_value, fixed_value, fixed_value, fixed_value, fixed_value + with open(power_fname, "w") as f: + print("# HELP node_hwmon_power_watt Hardware monitor for power consumption", file=f) + print("# TYPE node_hwmon_power_watt gauge", file=f) + print(f'node_hwmon_power_watt{{chip="power_supply_bat1",sensor="power1"}} {hwmon}', file=f) + print("# HELP node_rapl_package_joules_total Current RAPL package value in joules", file=f) + print("# TYPE node_rapl_package_joules_total counter", file=f) + print(f'node_rapl_package_joules_total{{index="0",path="/sys/class/powercap/intel-rapl:0"}} {rapl_0}', file=f) + print(f'node_rapl_package_joules_total{{index="1",path="/sys/class/powercap/intel-rapl:1"}} {rapl_1}', file=f) + print("# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts", file=f) + print("# TYPE ipmi_dcmi_power_consumption_watts gauge", file=f) + print(f"ipmi_dcmi_power_consumption_watts {dcmi}", file=f) + print("# HELP ipmi_power_watts Power reading of the sensor in watts", file=f) + print("# TYPE ipmi_power_watts gauge", file=f) + print(f'ipmi_power_watts{{id="1",name="Pwr Consumption"}} {ipmi}', file=f) def createPath(new_dir): dir_name = os.path.join(os.getcwd(), "precooked_tests") @@ -151,27 +189,35 @@ def createTestMonotonic(): proc_pid = os.path.join(proc, str(pid)) net_dir = os.path.join(new_dir, "net") nvidia_dir = os.path.join(new_dir, "nvidia") + power_dir = os.path.join(new_dir, "power") os.mkdir(proc_pid) os.mkdir(net_dir) os.mkdir(nvidia_dir) + os.mkdir(power_dir) if i == 1: make_stat(pid, proc_pid, 5000000) make_io(proc_pid, 500) make_smaps(proc_pid, 5000) make_net(net_dir, 500000) make_nvidia(nvidia_dir, 50, 50) + make_machstat(proc, 50000) + make_power(power_dir, 500) elif i == 2: make_stat(pid, proc_pid, 10000000) make_io(proc_pid, 1000) make_smaps(proc_pid, 10000) make_net(net_dir, 1000000) make_nvidia(nvidia_dir, 100, 100) + make_machstat(proc, 100000) + make_power(power_dir, 1000) else: make_stat(pid, proc_pid, 2000000) make_io(proc_pid, 200) make_smaps(proc_pid, 2000) make_net(net_dir, 200000) make_nvidia(nvidia_dir, "-", 20) + make_machstat(proc, 20000) + make_power(power_dir, 200) gc.collect() time.sleep(interval) i += 1 @@ -198,11 +244,15 @@ def createTestRand(dir_name, iter): os.mkdir(proc_pid) os.mkdir(net_dir) os.mkdir(nvidia_dir) + os.mkdir(mach_dir) + os.mkdir(power_dir) make_stat(pid, proc_pid, 0, True) make_io(proc_pid, 0, True) make_smaps(proc_pid, 0, True) make_net(net_dir, 0, True) make_nvidia(nvidia_dir, 0, True) + make_mach(mach_dir, 0, True) + make_power(power_dir, 0, True) gc.collect() time.sleep(interval) diff --git a/package/scripts/precooked_tests/drop/1/power/ipmi.prom b/package/scripts/precooked_tests/drop/1/power/ipmi.prom new file mode 100644 index 0000000..3e2290d --- /dev/null +++ b/package/scripts/precooked_tests/drop/1/power/ipmi.prom @@ -0,0 +1,13 @@ +# HELP node_hwmon_power_watt Hardware monitor for power consumption +# TYPE node_hwmon_power_watt gauge +node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 500 +# HELP node_rapl_package_joules_total Current RAPL package value in joules +# TYPE node_rapl_package_joules_total counter +node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 500 +node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 500 +# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts +# TYPE ipmi_dcmi_power_consumption_watts gauge +ipmi_dcmi_power_consumption_watts 500 +# HELP ipmi_power_watts Power reading of the sensor in watts +# TYPE ipmi_power_watts gauge +ipmi_power_watts{id="1",name="Pwr Consumption"} 500 diff --git a/package/scripts/precooked_tests/drop/1/proc/stat b/package/scripts/precooked_tests/drop/1/proc/stat new file mode 100644 index 0000000..7d47cd9 --- /dev/null +++ b/package/scripts/precooked_tests/drop/1/proc/stat @@ -0,0 +1 @@ +cpu 50000 50000 50000 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/package/scripts/precooked_tests/drop/2/power/ipmi.prom b/package/scripts/precooked_tests/drop/2/power/ipmi.prom new file mode 100644 index 0000000..264e28a --- /dev/null +++ b/package/scripts/precooked_tests/drop/2/power/ipmi.prom @@ -0,0 +1,13 @@ +# HELP node_hwmon_power_watt Hardware monitor for power consumption +# TYPE node_hwmon_power_watt gauge +node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 1000 +# HELP node_rapl_package_joules_total Current RAPL package value in joules +# TYPE node_rapl_package_joules_total counter +node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 1000 +node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 1000 +# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts +# TYPE ipmi_dcmi_power_consumption_watts gauge +ipmi_dcmi_power_consumption_watts 1000 +# HELP ipmi_power_watts Power reading of the sensor in watts +# TYPE ipmi_power_watts gauge +ipmi_power_watts{id="1",name="Pwr Consumption"} 1000 diff --git a/package/scripts/precooked_tests/drop/2/proc/stat b/package/scripts/precooked_tests/drop/2/proc/stat new file mode 100644 index 0000000..dd6a991 --- /dev/null +++ b/package/scripts/precooked_tests/drop/2/proc/stat @@ -0,0 +1 @@ +cpu 100000 100000 100000 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/package/scripts/precooked_tests/drop/3/power/ipmi.prom b/package/scripts/precooked_tests/drop/3/power/ipmi.prom new file mode 100644 index 0000000..9466571 --- /dev/null +++ b/package/scripts/precooked_tests/drop/3/power/ipmi.prom @@ -0,0 +1,13 @@ +# HELP node_hwmon_power_watt Hardware monitor for power consumption +# TYPE node_hwmon_power_watt gauge +node_hwmon_power_watt{chip="power_supply_bat1",sensor="power1"} 200 +# HELP node_rapl_package_joules_total Current RAPL package value in joules +# TYPE node_rapl_package_joules_total counter +node_rapl_package_joules_total{index="0",path="/sys/class/powercap/intel-rapl:0"} 200 +node_rapl_package_joules_total{index="1",path="/sys/class/powercap/intel-rapl:1"} 200 +# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts +# TYPE ipmi_dcmi_power_consumption_watts gauge +ipmi_dcmi_power_consumption_watts 200 +# HELP ipmi_power_watts Power reading of the sensor in watts +# TYPE ipmi_power_watts gauge +ipmi_power_watts{id="1",name="Pwr Consumption"} 200 diff --git a/package/scripts/precooked_tests/drop/3/proc/stat b/package/scripts/precooked_tests/drop/3/proc/stat new file mode 100644 index 0000000..6b0b2b0 --- /dev/null +++ b/package/scripts/precooked_tests/drop/3/proc/stat @@ -0,0 +1 @@ +cpu 20000 20000 20000 0 0 0 0 0 0 0 \ No newline at end of file diff --git a/package/src/machmon.cpp b/package/src/machmon.cpp index f1dc16f..39346ac 100644 --- a/package/src/machmon.cpp +++ b/package/src/machmon.cpp @@ -30,9 +30,8 @@ machmon::machmon() : mach_stats{} { // Read the first line of /proc/stat and return {utime, stime, ntime} -static std::tuple read_proc_stat( - const std::string& /*read_path*/) { - const std::string stat_path = "/proc/stat"; +static std::tuple read_proc_stat(const std::string read_path = "") { + const std::string stat_path = read_path + "/proc/stat"; std::ifstream proc_stat{stat_path}; if (!proc_stat) { throw std::runtime_error("Cannot open " + stat_path); @@ -67,9 +66,9 @@ void machmon::update_stats(const std::vector& /*pids*/, } const long clk = sysconf(_SC_CLK_TCK); - mach_stats.at("utime_total").set_value((utime - initial_utime) / clk); - mach_stats.at("stime_total").set_value((stime - initial_stime) / clk); - mach_stats.at("ntime_total").set_value((ntime - initial_ntime) / clk); + mach_stats.at("utime_total").set_value(std::max(0L, utime - initial_utime) / clk); + mach_stats.at("stime_total").set_value(std::max(0L, stime - initial_stime) / clk); + mach_stats.at("ntime_total").set_value(std::max(0L, ntime - initial_ntime) / clk); } // Return the summed counters diff --git a/package/src/powermon.cpp b/package/src/powermon.cpp index 9eca30b..f01f578 100644 --- a/package/src/powermon.cpp +++ b/package/src/powermon.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -37,38 +38,49 @@ static size_t curl_write_cb(char* ptr, size_t size, size_t nmemb, return new_size; } -// Constructor; uses RAII pattern to be valid after construction +// Constructor; uses RAII pattern to be valid +// after construction powermon::powermon() : power_stats{} { log_init(MONITOR_NAME); #undef MONITOR_NAME + + valid = false; // Default value, only set to true if everything works -// Global init for curl, explicitly + // Global init for curl, explicitly if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) { error("Failed to initialise libcurl"); return; } + // Initialize parameters for (const auto& param : params) { + // RAPL values are monotonic, so we set the flag to true. Else, false. if (param.get_name() == "rapl_1" || param.get_name() == "rapl_2") { - - // RAPL values are monotonic, so we set the flag to true power_stats.emplace(param.get_name(), prmon::monitored_value(param, true)); } else { power_stats.emplace(param.get_name(), prmon::monitored_value(param, false)); } } - valid = false; - // URLs from the environment variable PROMETHEUS_LOCAL_URL, if it exists - // Expected format: "http://localhost:9100/metrics,http://localhost:9290/metrics" - const char* env_string = std::getenv("PROMETHEUS_LOCAL_URL"); + // Function to check urls/directories and store them + auto add_entry = [&](const std::string& entry){ + if (entry.empty()) return; + if (entry.find("://") != std::string::npos) + metrics_urls.push_back(entry); + else + metrics_dirs.push_back(entry); + }; + // PROMETHEUS_LOCAL_URL: comma separated list of exporter URLs or directories + // Expected format: "http://localhost:9100/metrics,/node_exporter/ipmi.prom" + const char* env_string = std::getenv("PROMETHEUS_LOCAL_URL"); + + // Read through env_string, push each to either url or directory if (env_string) { unsigned pos{0}, start{0}; while (true) { if (env_string[pos] == ',' || env_string[pos] == '\0') { - if (pos > start) - metrics_urls.push_back(std::string(env_string + start, pos - start)); + add_entry(std::string(env_string + start, pos - start)); if (env_string[pos] == '\0') break; start = ++pos; } else { @@ -77,51 +89,49 @@ powermon::powermon() : power_stats{} { } } - // Use default values if no url was provided in the environment variable - if (metrics_urls.empty()) metrics_urls = default_metrics_urls; - + // Use default urls if no url was provided in the environment variable if (metrics_urls.empty()) { - debug("No exporter URLs provided, powermon disabled"); - return; - } - - bool reachable = false; + metrics_urls = default_metrics_urls; + debug("No exporter URLs provided, using default powermon ports."); + }; - std::vector dcmi, rapl, ipmi, hwmon; - for (const auto& url : metrics_urls) { - try { - collect_lines(fetch_metrics(url), dcmi, rapl, ipmi, hwmon); - reachable = true; - } catch (const std::exception& e) { - debug("powermon: endpoint " + url + " not usable: " + e.what()); - } - } + // Use default dirs if no dir was provided in the environment variable + if (metrics_dirs.empty()) { + metrics_dirs = default_metrics_dirs; + debug("No power directories provided, using default powermon dirs."); + }; - if (!reachable) { - info("Cannot connect to any exporter endpoint. powermon disabled."); + // Check at least one endpoint answers + const std::vector bodies = gather_bodies(""); + if (bodies.empty()){ + info("Cannot read power metrics from any exporter or directory." + "powermon disabled."); return; } + + // Check at least a metric is available + std::vector dcmi, rapl, ipmi, hwmon; + for (const auto& body : bodies) collect_lines(body, dcmi, rapl, ipmi, hwmon); if (dcmi.empty() && rapl.empty() && ipmi.empty() && hwmon.empty()) { info("Exporter reachable but none of the dcmi/rapl/ipmi/hwmon metrics " "were found. powermon disabled."); return; } - + + // If none of the errors are caught, then valid is set to true. valid = true; } // Destructor: closes libcurl global state -powermon::~powermon() { - curl_global_cleanup(); -} +powermon::~powermon() { curl_global_cleanup(); } -// Fetch Prometheus text from the ctt URL +// Fetch Prometheus text from the ctt URL. Returns "" on any failure. std::string powermon::fetch_metrics(const std::string& url) { std::string response{}; CURL* curl = curl_easy_init(); if (!curl) { error("Failed to initialise libcurl, curl_easy_init() returned NULL"); - return response; + return {}; } // Define a function to check if any response is != 0 @@ -137,28 +147,105 @@ std::string powermon::fetch_metrics(const std::string& url) { return true; }; + // Configure curl options. If any configuration fails, clean and return. if (!set_ok(CURLOPT_URL, url.c_str()) || !set_ok(CURLOPT_WRITEFUNCTION, curl_write_cb) || !set_ok(CURLOPT_WRITEDATA, &response) || !set_ok(CURLOPT_MAXFILESIZE_LARGE, kMaxResponseSize) || +#if LIBCURL_VERSION_NUM >= 0x075500 // 7.85.0 !set_ok(CURLOPT_PROTOCOLS_STR, "http") || +#else + !set_ok(CURLOPT_PROTOCOLS, static_cast(CURLPROTO_HTTP)) || +#endif !set_ok(CURLOPT_CONNECTTIMEOUT, 1L) || !set_ok(CURLOPT_TIMEOUT, 2L) || - !set_ok(CURLOPT_PROXY, "")) { - CURLcode res = curl_easy_perform(curl); + !set_ok(CURLOPT_PROXY, "") || + !set_ok(CURLOPT_NOSIGNAL, 1L)) { curl_easy_cleanup(curl); - return response; + return {}; }; - return {}; + + // Execute curl, check filesize + CURLcode curl_response = curl_easy_perform(curl); + if (curl_response == CURLE_FILESIZE_EXCEEDED || curl_response == CURLE_WRITE_ERROR) { + error("Response from " + url + "exceeds the " + + std::to_string(kMaxResponseSize) + "byte limit, discarding it"); + return {}; + } + return response; }; +// Read Prometheus text from a text file. +std::string powermon::read_metrics_file(const std::string& path){ + std::ifstream f{path}; + if (!f) return {}; // Missing file + + f.seekg(0, std::ios::end); + const std::streamoff size = f.tellg(); + if (size < 0 || size > kMaxResponseSize) { + error("File " + path + " exceeds the " + std::to_string(kMaxResponseSize) + + " byte limit, ignoring it"); + return {}; + } + f.seekg(0); + + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +// Collect Prometheus-style text from, in order of priority: +// 1. exporter URLs (PROMETHEUS_LOCAL_URL or default) +// 2. directories from PROMETHEUS_LOCAL_URL +// 3. default directory /power +// First source that works is used. +std::vector powermon::gather_bodies(const std::string& read_path){ + std::vector bodies; + + // 1. URLs (skipped if locked to directories) + if (source != Source::Files) { + for (const auto& url : metrics_urls) { + const std::string body = fetch_metrics(url); + if (!body.empty()) bodies.push_back(body); + } + if(!bodies.empty()) { + if (source == Source::Undecided){ + info("powermon: reading metrics from exporter URLs"); + source = Source::Http; + return bodies; + } + } + // Locked to HTTP and failed. + if (source == Source::Http) return bodies; + debug("powermon: reading from URLs failed"); + } + + // 2. Directories (if URLs failed or directory chosen) + std::vector dirs = metrics_dirs; + for (const auto& dir : dirs) { + debug("powermon: testing directory " + dir); + const std::string body = read_metrics_file(dir); + if (!body.empty()) bodies.push_back(body); + if (!bodies.empty()) { + if (source == Source::Undecided){ + info("powermon: reading metrics from directory " + dir); + source = Source::Files; + break; + } + } else { + debug("powermon: directory " + dir + " did not contain any metric or could not be read."); + } + + } + return bodies; +} + // Parse a single Prometheus line: "metric_name{labels} value timestamp" // Returns the value if the line starts with metric_name, otherwise -1.0 double powermon::parse_metric_line(const std::string& line, const std::string& metric_name) const { - if (line.empty() || line[0] == '#') return -1.0; - - if (line.rfind(metric_name, 0) != 0) return -1.0; + if (line.empty() || line[0] == '#') return -1.0; // Skip header or empty lines + if (line.rfind(metric_name, 0) != 0) return -1.0; // Skip lines without metric_name char next = line[metric_name.size()]; if (next != '{' && next != ' ' && next != '\t') return -1.0; @@ -174,7 +261,8 @@ double powermon::parse_metric_line(const std::string& line, try { const double val = std::stod(line.substr(value_start)); - if (std::isnan(val) || std::isinf(val)) return -1.0; // Return 1 if val is NaN or Inf + // Return -1 if val is NaN, Inf, negative or over the maximum + if (std::isnan(val) || std::isinf(val)) return -1.0; if (val < 0.0) return -1.0; if (val >= static_cast(std::numeric_limits::max())) return -1.0; return val; @@ -183,7 +271,7 @@ double powermon::parse_metric_line(const std::string& line, } } - +// Extract the rellevant lines from the text body void powermon::collect_lines(const std::string& body, std::vector& dcmi, std::vector& rapl, @@ -199,40 +287,33 @@ void powermon::collect_lines(const std::string& body, dcmi.push_back(line); if (lower.find("node_rapl_package_joules_total") != std::string::npos) rapl.push_back(line); - if (lower.find("ipmi_power_watts") != std::string::npos) + if (lower.find("ipmi_power_watts") != std::string::npos || + lower.find("ipmi_power_current_watts") != std::string::npos) ipmi.push_back(line); - if (lower.find("node_hwmon_power_watt") != std::string::npos) - hwmon.push_back(line); - else if (lower.find("node_hwmon_power_average_watt") != std::string::npos) + if (lower.find("node_hwmon_power_watt") != std::string::npos || + lower.find("node_hwmon_power_average_watt") != std::string::npos) hwmon.push_back(line); } } void powermon::update_stats(const std::vector& /*pids*/, - const std::string /*read_path*/) { - // Collect matching lines from all URLs - std::vector dcmi_lines{}; - std::vector rapl_lines{}; - std::vector ipmi_lines{}; - std::vector hwmon_lines{}; - - int failed_urls = 0; - for (const auto& url : metrics_urls) { - try { - collect_lines(fetch_metrics(url), dcmi_lines, rapl_lines, - ipmi_lines, hwmon_lines); - } catch (...) { - ++failed_urls; - } - } + const std::string read_path) { - if (failed_urls == (int)metrics_urls.size()) { - error("Cannot connect to any exporter endpoint, disabling powermon"); + // Attempt to read from each metrics_url + const std::vector bodies = gather_bodies(read_path); + if (bodies.empty()) { + info("No power metrics from any exporter or directory, disabling powermon"); valid = false; return; } + // Collect matching lines from all sources + std::vector dcmi_lines{}, rapl_lines{}, ipmi_lines{}, hwmon_lines{}; + for (const auto& body : bodies) { + collect_lines(body, dcmi_lines, rapl_lines, ipmi_lines, hwmon_lines); + } + // DCMI: sum all matching lines double dcmi = 0.0; try { @@ -263,6 +344,7 @@ void powermon::update_stats(const std::vector& /*pids*/, try { if (!ipmi_lines.empty()) { double val = parse_metric_line(ipmi_lines.back(), "ipmi_power_watts"); + if (val < 0.0) val = parse_metric_line(ipmi_lines.back(), "ipmi_power_current_watts"); if (val >= 0.0) ipmi = val; } } catch (...) { ipmi = 0.0; } diff --git a/package/src/powermon.h b/package/src/powermon.h index 971e2b5..f4f9412 100644 --- a/package/src/powermon.h +++ b/package/src/powermon.h @@ -27,6 +27,26 @@ class powermon final : public Imonitor, public MessageBase { {"rapl_2", "J", "RAPL package 2 energy in joules"} }; + // Default urls to look at + const std::vector default_metrics_urls{ + "http://localhost:9100/metrics", + "http://localhost:9290/metrics" + }; + std::vector metrics_urls; + + // Directories to look at + const std::vector default_metrics_dirs{ + "/power/ipmi.prom" + }; + std::vector metrics_dirs; + + // Source of metrics, decided first time data is found + enum class Source {Undecided, Http, Files}; + Source source{Source::Undecided}; + +// // Check and choose data source +// void source_check(); + // Extract lines from the Prometheus text output for each metric type void collect_lines(const std::string& body, std::vector& dcmi, @@ -38,13 +58,6 @@ class powermon final : public Imonitor, public MessageBase { prmon::monitored_list power_stats; - const std::vector default_metrics_urls{ - "http://localhost:9100/metrics", - "http://localhost:9290/metrics" - }; - - std::vector metrics_urls; - std::string fetch_metrics(const std::string& url); double parse_metric_line(const std::string& line, const std::string& metric_name) const; @@ -55,12 +68,19 @@ class powermon final : public Imonitor, public MessageBase { void update_stats(const std::vector& pids, const std::string read_path = ""); - + + // These are the stat getter methods which retrieve current statistics prmon::monitored_value_map const get_text_stats(); prmon::monitored_value_map const get_json_total_stats(); prmon::monitored_average_map const get_json_average_stats( unsigned long long elapsed_clock_ticks); prmon::parameter_list const get_parameter_list(); + + // Additional function to read from a directory instead of curl + std::string read_metrics_file(const std::string& read_path = ""); + std::vector gather_bodies(const std::string& read_path); + + // This is the hardware information getter that runs once void const get_hardware_info(nlohmann::json& hw_json); void const get_unit_info(nlohmann::json& unit_json); bool const is_valid() { return valid; } From ebefc0407baea4f60d9d2334c13362951f065276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Thu, 29 Oct 2026 16:23:33 +0100 Subject: [PATCH 12/14] CodeFactor, flake8 style corrections --- package/scripts/precook_test.py | 18 +++++++++++------- .../scripts/precooked_tests/drop/3/nvidia/smi | 2 +- package/src/powermon.cpp | 2 -- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index 032053a..0d2f2ad 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -120,24 +120,26 @@ def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, print(param, file=f, end=" ") print("", file=f) + def make_machstat(proc_mach, fixed_value, rand=False): mach_fname = os.path.join(proc_mach, "stat") lim = 2**64 - 1 if rand: - user, nice, system = random.randint(1,lim) + user, nice, system = random.randint(1, lim) else: user, nice, system = fixed_value, fixed_value, fixed_value with open(mach_fname, "w") as f: - print("cpu", file = f, end = " ") # Relevant line + print("cpu", file=f, end=" ") # Relevant line # 1 utime - Time in user mode # 2 ntime - Time in nice mode # 3 stime - Time in system mode - print(f"{user} {nice} {system}", file = f, end = " ") + print(f"{user} {nice} {system}", file=f, end=" ") # 4 - 10 are never used, set to 0 - print("0 " * 7, file = f, end = "") + print("0 "*7, file=f, end="") + -def make_power(power_dir, fixed_value, rand = False): +def make_power(power_dir, fixed_value, rand=False): power_fname = os.path.join(power_dir, "ipmi.prom") if rand: dcmi, ipmi, hwmon = (random.randint(0, 1000) for _ in range(3)) @@ -159,6 +161,7 @@ def make_power(power_dir, fixed_value, rand = False): print("# TYPE ipmi_power_watts gauge", file=f) print(f'ipmi_power_watts{{id="1",name="Pwr Consumption"}} {ipmi}', file=f) + def createPath(new_dir): dir_name = os.path.join(os.getcwd(), "precooked_tests") if not os.path.isdir(dir_name): @@ -241,17 +244,18 @@ def createTestRand(dir_name, iter): proc_pid = os.path.join(proc, str(pid)) net_dir = os.path.join(new_dir, "net") nvidia_dir = os.path.join(new_dir, "nvidia") + power_dir = os.path.join(new_dir, "power") os.mkdir(proc_pid) os.mkdir(net_dir) os.mkdir(nvidia_dir) - os.mkdir(mach_dir) + os.mkdir(proc) os.mkdir(power_dir) make_stat(pid, proc_pid, 0, True) make_io(proc_pid, 0, True) make_smaps(proc_pid, 0, True) make_net(net_dir, 0, True) make_nvidia(nvidia_dir, 0, True) - make_mach(mach_dir, 0, True) + make_mach(proc, 0, True) make_power(power_dir, 0, True) gc.collect() time.sleep(interval) diff --git a/package/scripts/precooked_tests/drop/3/nvidia/smi b/package/scripts/precooked_tests/drop/3/nvidia/smi index 07cbf9c..c17b14f 100644 --- a/package/scripts/precooked_tests/drop/3/nvidia/smi +++ b/package/scripts/precooked_tests/drop/3/nvidia/smi @@ -1 +1 @@ -0 1729 G - - - - - - 20 0 python3 +0 1729 G - - - - - - 20 0 python3 diff --git a/package/src/powermon.cpp b/package/src/powermon.cpp index f01f578..1e8ae10 100644 --- a/package/src/powermon.cpp +++ b/package/src/powermon.cpp @@ -51,7 +51,6 @@ powermon::powermon() : power_stats{} { error("Failed to initialise libcurl"); return; } - // Initialize parameters for (const auto& param : params) { // RAPL values are monotonic, so we set the flag to true. Else, false. @@ -299,7 +298,6 @@ void powermon::collect_lines(const std::string& body, void powermon::update_stats(const std::vector& /*pids*/, const std::string read_path) { - // Attempt to read from each metrics_url const std::vector bodies = gather_bodies(read_path); if (bodies.empty()) { From 89dcd916956a0bb5476cd19b9d84b9ea4b3a476a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Thu, 29 Oct 2026 16:31:36 +0100 Subject: [PATCH 13/14] precook_test correction, typo in make_test --- package/scripts/precook_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index 0d2f2ad..1ec0bf3 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -255,7 +255,7 @@ def createTestRand(dir_name, iter): make_smaps(proc_pid, 0, True) make_net(net_dir, 0, True) make_nvidia(nvidia_dir, 0, True) - make_mach(proc, 0, True) + make_machstat(proc, 0, True) make_power(power_dir, 0, True) gc.collect() time.sleep(interval) From 65f2eb11670b28963125ba02391e615038cc0d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez=20Garc=C3=ADa?= Date: Thu, 29 Oct 2026 17:36:13 +0100 Subject: [PATCH 14/14] Proper formatting precook_test --- package/scripts/precook_test.py | 54 +++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index 1ec0bf3..fc9ccfe 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -93,8 +93,9 @@ def make_net(proc_net, fixed_value, rand=False): print(random.randint(0, net_lim) if rand else fixed_value, file=f) -def make_nvidia(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, - rand_sm_mem=False, rand_fb=False): +def make_nvidia( + proc_nvidia, fixed_value_sm_mem, fixed_value_fb, rand_sm_mem=False, rand_fb=False +): # idx print(proc_nvidia, fixed_value_sm_mem, fixed_value_fb, rand_sm_mem, rand_fb) smi_fname = os.path.join(proc_nvidia, "smi") @@ -136,7 +137,7 @@ def make_machstat(proc_mach, fixed_value, rand=False): print(f"{user} {nice} {system}", file=f, end=" ") # 4 - 10 are never used, set to 0 - print("0 "*7, file=f, end="") + print("0 " * 7, file=f, end="") def make_power(power_dir, fixed_value, rand=False): @@ -145,21 +146,50 @@ def make_power(power_dir, fixed_value, rand=False): dcmi, ipmi, hwmon = (random.randint(0, 1000) for _ in range(3)) rapl_0, rapl_1 = (random.randint(0, 1000000) for _ in range(3)) else: - dcmi, ipmi, hwmon, rapl_0, rapl_1 = fixed_value, fixed_value, fixed_value, fixed_value, fixed_value + dcmi = ipmi = hwmon = rapl_0 = rapl_1 = fixed_value + with open(power_fname, "w") as f: - print("# HELP node_hwmon_power_watt Hardware monitor for power consumption", file=f) + print( + "# HELP node_hwmon_power_watt " "Hardware monitor for power consumption", + file=f, + ) print("# TYPE node_hwmon_power_watt gauge", file=f) - print(f'node_hwmon_power_watt{{chip="power_supply_bat1",sensor="power1"}} {hwmon}', file=f) - print("# HELP node_rapl_package_joules_total Current RAPL package value in joules", file=f) + print( + "node_hwmon_power_watt" + f'{{chip="power_supply_bat1",sensor="power1"}} {hwmon}', + file=f, + ) + + print( + "# HELP node_rapl_package_joules_total " + "Current RAPL package value in joules", + file=f, + ) print("# TYPE node_rapl_package_joules_total counter", file=f) - print(f'node_rapl_package_joules_total{{index="0",path="/sys/class/powercap/intel-rapl:0"}} {rapl_0}', file=f) - print(f'node_rapl_package_joules_total{{index="1",path="/sys/class/powercap/intel-rapl:1"}} {rapl_1}', file=f) - print("# HELP ipmi_dcmi_power_consumption_watts Current power consumption in watts", file=f) + print( + "node_rapl_package_joules_total" + '{index="0",path="/sys/class/powercap/intel-rapl:0"}' + f" {rapl_0}", + file=f, + ) + print( + "node_rapl_package_joules_total" + '{index="1",path="/sys/class/powercap/intel-rapl:1"}' + f" {rapl_1}", + file=f, + ) + + print( + "# HELP ipmi_dcmi_power_consumption_watts " + "Current power consumption in watts", + file=f, + ) print("# TYPE ipmi_dcmi_power_consumption_watts gauge", file=f) print(f"ipmi_dcmi_power_consumption_watts {dcmi}", file=f) - print("# HELP ipmi_power_watts Power reading of the sensor in watts", file=f) + + print("# HELP ipmi_power_watts " "Power reading of the sensor in watts", file=f) print("# TYPE ipmi_power_watts gauge", file=f) - print(f'ipmi_power_watts{{id="1",name="Pwr Consumption"}} {ipmi}', file=f) + print('ipmi_power_watts{id="1",name="Pwr Consumption"}' f" {ipmi}", file=f) def createPath(new_dir):