From 31dd43e522659308c503706c297bc1a876b9cf3e Mon Sep 17 00:00:00 2001 From: stewit <> Date: Tue, 6 Oct 2026 13:44:31 +0200 Subject: [PATCH 01/15] fall back to inherited metric metadata Metric metadata accessors now use the properties a metric inherits from objects higher up in an asset hierarchy (dict under "inherited" in the metric object of the current metadata convention) if the metric itself does not provide the requested information. Can be deactivated via the keyword argument include_inherited=False. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 + docs/index.html | 117 +++++++++++++++++-------- docs/searchindex.js | 2 +- src/hdhelpers/metadata/_helpers.py | 21 +++-- src/hdhelpers/metadata/_specs.py | 40 +++++++-- src/hdhelpers/metadata/helpers.py | 133 ++++++++++++++++++++--------- tests/metadata/test_metadata.py | 86 +++++++++++++++++++ 7 files changed, 315 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7705fb..c3e93ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## [Unreleased] +- metric metadata accessors fall back to the properties a metric inherits (under "inherited" in the metric metadata of the current convention), can be deactivated via keyword argument include_inherited=False + ## [0.0.9] - 2026-09-10 - singletsframe support diff --git a/docs/index.html b/docs/index.html index a8ec446..2513aee 100644 --- a/docs/index.html +++ b/docs/index.html @@ -74,11 +74,14 @@

Functions
-hdhelpers.metadata.get_display_names(multitsframe)
+hdhelpers.metadata.get_display_names(multitsframe, *, include_inherited=True)

Gets display names of the MTSF metrics from the metadata

Parameters:
-

multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

+
    +

  • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of metrics containing the display names. @@ -119,11 +122,14 @@

Functions
-hdhelpers.metadata.get_measurements(multitsframe)
+hdhelpers.metadata.get_measurements(multitsframe, *, include_inherited=True)

Gets measurement (type) of the MTSF metrics from the metadata

Parameters:
-

multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

+
    +

  • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of metrics containing the measurement (type) of the MTSF metrics. @@ -149,7 +155,7 @@

Functions
-hdhelpers.metadata.get_metric_info(multitsframe, metric_info)
+hdhelpers.metadata.get_metric_info(multitsframe, metric_info, *, include_inherited=True)

Gets a dictionary of metadata associated to metrics.

In contrast to metadata associated to concrete value dimensions, this function abstracts access to metadata associated to the underlying metric.

@@ -158,6 +164,7 @@

Functions

  • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

  • metric_info (str | Spec) – Name of information to retrieve. Note that metric_info is interpreted as a glom Spec.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

Returns:
@@ -212,11 +219,14 @@

Functions
-hdhelpers.metadata.get_names(multitsframe)
+hdhelpers.metadata.get_names(multitsframe, *, include_inherited=True)

Gets names of the MTSF metrics from the metadata

Parameters:
-

multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

+
    +

  • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of metrics containing the names. If the name is not present for a metric the corresponding value is None.

@@ -294,11 +304,14 @@

Functions
-hdhelpers.metadata.get_series_display_name(series)
+hdhelpers.metadata.get_series_display_name(series, *, include_inherited=True)

Gets display name of the series from metadata

Parameters:
-

series (pd.Series) – Series with metadata following the convention.

+
    +

  • series (pd.Series) – Series with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Returns the display name of the value. @@ -323,7 +336,7 @@

Functions
-hdhelpers.metadata.get_series_info(series, value_dim_info)
+hdhelpers.metadata.get_series_info(series, value_dim_info, *, include_inherited=True)

Gets an arbitrary series info

Since a series has only one value dimension named “value”, this information is equivalent to information on the metric.

@@ -332,6 +345,7 @@

Functions

  • series (pd.Series) – Series with metadata following the convention.

  • value_dim_info (str | Spec) – Name of information to retrieve. Note that value_dim_info is interpreted as a glom Spec.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

Returns:
@@ -354,11 +368,14 @@

Functions
-hdhelpers.metadata.get_series_measurement(series)
+hdhelpers.metadata.get_series_measurement(series, *, include_inherited=True)

Gets measurement (type) of the Series from metadata

Parameters:
-

series (pd.Series) – Series with metadata following the convention.

+
    +

  • series (pd.Series) – Series with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Returns the measurement (type) of the value. @@ -382,11 +399,14 @@

Functions
-hdhelpers.metadata.get_series_name(series)
+hdhelpers.metadata.get_series_name(series, *, include_inherited=True)

Gets name of the series from metadata

Parameters:
-

series (pd.Series) – Series with metadata following the convention.

+
    +

  • series (pd.Series) – Series with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Returns the name of the value. @@ -426,11 +446,14 @@

Functions
-hdhelpers.metadata.get_series_short_display_name(series)
+hdhelpers.metadata.get_series_short_display_name(series, *, include_inherited=True)

Gets short display name of the Series from metadata

Parameters:
-

series (pd.Series) – Series with metadata following the convention.

+
    +

  • series (pd.Series) – Series with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Returns the short display name of the value. @@ -455,11 +478,14 @@

Functions
-hdhelpers.metadata.get_series_unit(series)
+hdhelpers.metadata.get_series_unit(series, *, include_inherited=True)

Gets name of the series from metadata

Parameters:
-

series (pd.Series) – Series with metadata following the convention.

+
    +

  • series (pd.Series) – Series with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Returns the unit of series. @@ -496,11 +522,14 @@

Functions
-hdhelpers.metadata.get_short_display_names(multitsframe)
+hdhelpers.metadata.get_short_display_names(multitsframe, *, include_inherited=True)

Gets short display names of the MTSF metrics from the metadata

Parameters:
-

multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

+
    +

  • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of metrics containing the short display names. @@ -544,12 +573,15 @@

Functions
-hdhelpers.metadata.get_singlets_display_names(singletsframe)
+hdhelpers.metadata.get_singlets_display_names(singletsframe, *, include_inherited=True)

Gets display names of the value dimensions of a SingleTSFrame from metadata

Falls back to the name of the value dimension if no display name is given.

Parameters:
-

singletsframe (pd.DataFrame) – STSF with metadata following the convention.

+
    +

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of value dimensions containing the display names. If neither display name nor name is present for a value dimension the corresponding value is None.

@@ -585,7 +617,7 @@

Functions
-hdhelpers.metadata.get_singlets_info(singletsframe, value_dim_info)
+hdhelpers.metadata.get_singlets_info(singletsframe, value_dim_info, *, include_inherited=True)

Gets an arbitrary info of the value dimensions of a SingleTSFrame

A SingleTSFrame (STSF) holds exactly one metric - like a Series - but arbitrarily many value dimensions - like a MultiTSFrame. Therefore, in contrast to the MTSF functions, @@ -598,6 +630,7 @@

Functions

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

  • value_dim_info (str | Spec) – Name of information to retrieve. Note that value_dim_info is interpreted as a glom Spec.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

Returns:
@@ -634,11 +667,14 @@

Functions
-hdhelpers.metadata.get_singlets_measurements(singletsframe)
+hdhelpers.metadata.get_singlets_measurements(singletsframe, *, include_inherited=True)

Gets measurements (types) of the value dimensions of a SingleTSFrame from metadata

Parameters:
-

singletsframe (pd.DataFrame) – STSF with metadata following the convention.

+
    +

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of value dimensions containing the measurements. If “measurement” is not given for a value dimension the corresponding value is None.

@@ -670,7 +706,7 @@

Functions
-hdhelpers.metadata.get_singlets_metric_info(singletsframe, metric_info)
+hdhelpers.metadata.get_singlets_metric_info(singletsframe, metric_info, *, include_inherited=True)

Gets metadata associated to the single metric of a SingleTSFrame

Counterpart of get_metric_info() for SingleTSFrames: instead of a dictionary keyed by metric this directly returns the requested information for the one metric.

@@ -679,6 +715,7 @@

Functions

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

  • metric_info (str | Spec) – Name of information to retrieve. Note that metric_info is interpreted as a glom Spec.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

Returns:
@@ -712,11 +749,14 @@

Functions
-hdhelpers.metadata.get_singlets_names(singletsframe)
+hdhelpers.metadata.get_singlets_names(singletsframe, *, include_inherited=True)

Gets names of the value dimensions of a SingleTSFrame from metadata

Parameters:
-

singletsframe (pd.DataFrame) – STSF with metadata following the convention.

+
    +

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of value dimensions containing the names. If the name is not present for a value dimension the corresponding value is None.

@@ -752,13 +792,16 @@

Functions
-hdhelpers.metadata.get_singlets_short_display_names(singletsframe)
+hdhelpers.metadata.get_singlets_short_display_names(singletsframe, *, include_inherited=True)

Gets short display names of the value dimensions of a SingleTSFrame from metadata

Falls back to the display name and then to the name of the value dimension if no short display name is given.

Parameters:
-

singletsframe (pd.DataFrame) – STSF with metadata following the convention.

+
    +

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of value dimensions containing the short display names. If none of short display name, display name and name is present for a value dimension the corresponding value is None.

@@ -794,11 +837,14 @@

Functions
-hdhelpers.metadata.get_singlets_units(singletsframe)
+hdhelpers.metadata.get_singlets_units(singletsframe, *, include_inherited=True)

Gets units of the value dimensions of a SingleTSFrame from metadata

Parameters:
-

singletsframe (pd.DataFrame) – STSF with metadata following the convention.

+
    +

  • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of value dimensions containing the units. If the unit is not present for a value dimension the corresponding value is None.

@@ -849,11 +895,14 @@

Functions
-hdhelpers.metadata.get_units(multitsframe)
+hdhelpers.metadata.get_units(multitsframe, *, include_inherited=True)

Gets unit of value dimensions in MTSF metrics from Metadata

Parameters:
-

multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

+
    +

  • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

    • +

  • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

    • +
Returns:

Dictionary of metrics containing the names of the value dimensions. diff --git a/docs/searchindex.js b/docs/searchindex.js index 48248e2..cdd0c31 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"000z":0,"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":1,"DO":0,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Our":0,"The":1,"This":1,"To":1,"We":0,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":1,"ambigu":1,"ani":1,"anoth":1,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":1,"background":1,"base":1,"behaviour":1,"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"can":[0,1],"carri":1,"case":1,"certain":1,"class":1,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":1,"column_nam":1,"compli":1,"compon":[0,1],"concret":1,"contain":1,"context":1,"contrast":1,"control":1,"convent":1,"convert":1,"convert_index":1,"correspond":[0,1],"counterpart":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":1,"date":1,"datetim":1,"de":1,"de_d":1,"def":0,"default":1,"default_titl":0,"defaultdict":1,"defin":[0,1],"design":[0,1],"determin":1,"dict":1,"dictionari":1,"dimens":1,"direct":[0,1],"display":1,"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"e":1,"ea1255":1,"eas":1,"eb6962":1,"edit":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":1,"error_cod":1,"especi":1,"europ":1,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":1,"external_second":1,"extra_inform":1,"extract":1,"fall":1,"fig":[0,1],"figur":[0,1],"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":1,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":1,"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":1,"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":1,"get_singlets_measur":1,"get_singlets_metric_info":1,"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":1,"get_unit":1,"get_value_dimension_info":1,"given":1,"global":1,"glom":1,"go":[0,1],"good":1,"graph_object":[0,1],"handl":1,"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"index":[0,1],"infer":1,"info":1,"inspir":1,"instal":1,"instead":1,"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":1,"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":1,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"look":0,"m":1,"main":0,"mani":1,"marker":0,"meaning":1,"measur":1,"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":1,"might":0,"miss":1,"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":1,"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"non":1,"none":1,"note":1,"object":1,"object_to_convert":1,"one":1,"onli":1,"oper":1,"option":1,"orang":1,"output":1,"packag":1,"panda":1,"paramet":1,"pd":1,"place":1,"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":1,"print":1,"provid":1,"provis":0,"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"repres":0,"request":1,"resolv":1,"resolve_color":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"s":[0,1],"say":0,"scatter":[0,1],"scheme":1,"second":1,"see":1,"sensibl":1,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":1,"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":1,"singl":1,"single_metr":1,"single_metric_dataset_metadata":0,"single_metric_metadata":[0,1],"singletsfram":1,"spec":1,"specifi":1,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":1,"streamlin":1,"string":1,"structured_metadata":[0,1],"stsf":1,"style":1,"surpress":1,"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":1,"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"use":1,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":1,"value_dimensions_shar":1,"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":1,"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whose":1,"will":1,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":1,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":1,"introduct":1,"metadata":[0,1],"plot":[0,1],"step":0,"tbd":0,"use":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"000z":0,"00z":1,"01t01":1,"01t08":0,"01t09":[],"01t10":[],"05t13":1,"06t13":1,"5s":[],"80b0ec":[],"A":1,"All":[],"DO":0,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Our":0,"That":[],"The":1,"This":1,"To":1,"We":0,"With":[],"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"activ":[],"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":1,"also":[],"ambigu":1,"ani":1,"anoth":1,"appli":[],"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"asset":1,"associ":1,"assum":1,"attr":1,"avail":1,"awar":[],"axe":1,"axi":0,"back":1,"background":1,"background_color":[],"base":1,"becaus":[],"behaviour":1,"berlin":1,"bool":1,"boolean":1,"brand":[],"by_metr":1,"c":1,"can":[0,1],"carri":1,"case":1,"certain":1,"chosen":[],"class":1,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":1,"column_nam":1,"communic":[],"compli":1,"compon":[0,1],"concret":1,"configur":[],"contain":1,"context":1,"contrast":1,"control":1,"convent":1,"convert":1,"convert_index":1,"convey":[],"correspond":[0,1],"counterpart":1,"current":[],"custom":1,"cut":[],"danger":[],"dark":1,"data":1,"datafram":1,"dataset_metadata":1,"date":1,"datetim":1,"de":1,"de_d":1,"def":0,"default":1,"default_titl":0,"default_unit":[],"defaultdict":1,"defin":[0,1],"design":[0,1],"detail":[],"determin":1,"dict":1,"dictionari":1,"dimens":1,"direct":[0,1],"display":1,"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"draw":[],"e":1,"ea1255":1,"eas":1,"eb6962":1,"edg":[],"edit":0,"els":[],"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":1,"error_cod":1,"error_color":[],"especi":1,"europ":1,"exact":1,"exampl":[0,1],"exclus":1,"execcontext":[],"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":1,"external_second":1,"extra_inform":1,"extract":1,"fail":[],"fall":1,"featur":[],"fig":[0,1],"figur":[0,1],"first":1,"flag":[],"follow":[0,1],"format":1,"four":[],"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":1,"generat":1,"german":1,"get":1,"get_and_pad_start_and_end_timestamp":[],"get_colors_from_plot_target_set":[],"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":1,"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":1,"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":1,"get_singlets_measur":1,"get_singlets_metric_info":1,"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":1,"get_unit":1,"get_value_dimension_info":1,"get_y_axis_label":[],"given":1,"global":1,"glom":1,"go":[0,1],"goal":[],"good":1,"graph_object":[0,1],"half":[],"handl":1,"happen":1,"hardwir":[],"helper":0,"helperexcept":1,"hetida":[0,1],"hide":[],"hierarchi":1,"higher":1,"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":1,"index":[0,1],"infer":1,"info":1,"info_color":[],"inform":[],"inherit":1,"inpir":[],"input":[],"insoformat":[],"inspir":1,"instal":1,"instead":1,"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"intend":[],"interpret":1,"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"keep":[],"key":1,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"load":[],"local":1,"look":0,"m":1,"magnitud":[],"main":0,"make":[],"mani":1,"marker":0,"meaning":1,"measur":1,"mention":[],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":1,"might":0,"miss":1,"mode":[],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":1,"naiv":[],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"non":1,"none":1,"note":1,"object":1,"object_to_convert":1,"one":1,"onli":1,"oper":1,"option":1,"orang":1,"order":[],"otherwis":[],"output":1,"packag":1,"panda":1,"paramet":1,"pars":[],"pass":[],"pd":1,"place":1,"platform":1,"pleas":1,"plot":[],"plot_target_set":1,"plot_target_styl":[],"plot_target_timezon":[],"plotly_fig_to_json_dict":[],"point":[],"possibl":1,"potenti":[],"precis":[],"present":1,"print":1,"properti":1,"provid":1,"provis":0,"purpos":[],"put":[],"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"reason":[],"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"reflect":[],"repres":0,"request":1,"resolv":1,"resolve_color":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"runtim":[],"s":[0,1],"say":0,"scatter":[0,1],"scheme":1,"second":1,"see":1,"sensibl":1,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_language_agnostic_datetime_tick":[],"sever":[],"shade":1,"short":1,"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":1,"singl":1,"single_metr":1,"single_metric_dataset_metadata":0,"single_metric_metadata":[0,1],"singletsfram":1,"spec":1,"specif":[],"specifi":1,"standard":[0,1],"start":[0,1],"start_pad":[],"state":1,"status":[],"status_color":[],"step":1,"str":1,"streamlin":1,"string":1,"structured_metadata":[0,1],"stsf":1,"style":1,"success_color":[],"sure":[],"surpress":1,"switch":[],"t":1,"take":[],"target":1,"tbd":1,"tech":[],"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"theme_agnostic_templ":[],"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"titl":[],"to_datetim":1,"to_timezon":1,"togeth":[],"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":1,"tupl":1,"turn":[],"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"usag":[],"use":1,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":1,"value_dimensions_shar":1,"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":1,"vision":0,"want":0,"warn_color":[],"water":0,"way":1,"well":1,"whether":1,"whose":1,"will":1,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":1,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"exampl":[],"except":1,"explan":[],"first":0,"full":[],"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":1,"introduct":1,"metadata":[0,1],"plot":[0,1],"step":0,"tbd":0,"use":0}}) \ No newline at end of file diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 0120981..e03a13b 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -33,7 +33,9 @@ def extract_from_metadata(metadata: Any, key: str, default: str | None = None) - def get_value_dimension_info( - multitsframe: pd.DataFrame | pd.Series, value_dim_info: str | Spec + multitsframe: pd.DataFrame | pd.Series, + value_dim_info: str | Spec, + include_inherited: bool = True, ) -> defaultdict[str, defaultdict[str, Any]]: """Obtain metadata info associated to the value dimensions of the metrics @@ -51,12 +53,15 @@ def get_value_dimension_info( dimension in the value_dimensions list under the metric, the global "value_dimensions_shared" field of the attrs object is searched for corresponding information. + If include_inherited, the properties a metric inherits are used as part of its metric + metadata, i.e. after the metric's own information but before "value_dimensions_shared". + If no information is found, None is set as value and is the default value of the inner default dict. For examples we refer to the corresponding unit tests (/tests/helpers/test_metadata.py). """ - spec = _specs.by_metric_key_by_val_dimension(value_dim_info) + spec = _specs.by_metric_key_by_val_dimension(value_dim_info, include_inherited) value_dimension_info_by_metric_by_value_dimension = glom(multitsframe.attrs, spec) return defaultdict( lambda: defaultdict(lambda: None), value_dimension_info_by_metric_by_value_dimension @@ -84,7 +89,7 @@ def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: def singlets_value_dimension_info( - singletsframe: pd.DataFrame, value_dim_info: str | Spec + singletsframe: pd.DataFrame, value_dim_info: str | Spec, include_inherited: bool = True ) -> defaultdict[str, Any]: """Metadata info of the value dimensions of the single metric of a SingleTSFrame @@ -100,13 +105,15 @@ def singlets_value_dimension_info( only the metric level is collapsed away. """ return _select_single_metric( - get_value_dimension_info(singletsframe, value_dim_info), + get_value_dimension_info(singletsframe, value_dim_info, include_inherited), singletsframe.attrs, empty=defaultdict(lambda: None), ) -def singlets_metric_info(singletsframe: pd.DataFrame, metric_info: str | Spec) -> Any: +def singlets_metric_info( + singletsframe: pd.DataFrame, metric_info: str | Spec, include_inherited: bool = True +) -> Any: """Metadata of the single metric of a SingleTSFrame Counterpart of get_metric_info for SingleTSFrames: instead of a mapping keyed by metric @@ -115,7 +122,9 @@ def singlets_metric_info(singletsframe: pd.DataFrame, metric_info: str | Spec) - not require metadata (see the metadata conventions documentation). """ try: - info_by_metric = glom(singletsframe.attrs, _specs.spec_by_metric_key(metric_info)) + info_by_metric = glom( + singletsframe.attrs, _specs.spec_by_metric_key(metric_info, include_inherited) + ) except GlomError: # no metric metadata present at all return None diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index e5749bf..71738f9 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -12,8 +12,29 @@ DISPLAY_NAME: Spec = Coalesce("display_name", "name", default=None) SHORT_DISPLAY_NAME: Spec = Coalesce("short_display_name", "display_name", "name", default=None) +# Key of the dict in a metric object holding the properties the metric inherits from objects +# higher up in an asset hierarchy, e.g. the latitude of the plant a sensor belongs to. +INHERITED = "inherited" -def by_metric_key_by_val_dimension(metadatum_key: str | Spec) -> Spec: + +def of_metric(metadatum_key: str | Spec, include_inherited: bool = True) -> Spec: + """Provides glom spec that extracts a metadatum from a metric object + + If include_inherited, the metric's inherited properties are used as fallback: the whole + metadatum_key spec is first applied to the metric itself and only if that yields None + or fails, it is applied to the INHERITED dict of the metric. So in a fallback chain + like DISPLAY_NAME an own "name" beats an inherited "display_name". + + metadatum_key can also be any glom spec. + """ + if not include_inherited: + return metadatum_key + return Coalesce(metadatum_key, (INHERITED, metadatum_key), skip=lambda x: x is None) + + +def by_metric_key_by_val_dimension( + metadatum_key: str | Spec, include_inherited: bool = True +) -> Spec: """Providesglom spec that extracts a metadatum by metric by value dimension The generated glom spec returns a defaultdict of defaultdicts: @@ -27,10 +48,13 @@ def by_metric_key_by_val_dimension(metadatum_key: str | Spec) -> Spec: Properly falls back to "value_dimensions_shared" metadata if a value_dimension is not given for a metric if its available there. + For the current metadata convention, properties the metric inherits are used as part of + the metric metadatum if include_inherited, see of_metric. + metadatum_key can also be any glom spec. """ return Coalesce( - _spec_new_convention(metadatum_key), + _spec_new_convention(metadatum_key, include_inherited), _spec_platform_convention_series_metric(metadatum_key), _spec_platform_convention_series_value_dim(metadatum_key), _spec_older_convention1_metric(metadatum_key), @@ -65,14 +89,14 @@ def _spec_defaults_by_value_dimension(metadatum_key: str | Spec) -> Spec: ) -def _spec_defaults_by_metric(metadatum_key: str | Spec) -> Spec: +def _spec_defaults_by_metric(metadatum_key: str | Spec, include_inherited: bool) -> Spec: return Coalesce( ( "metrics", Check(instance_of=list), _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( S.globals.metric_key, - Coalesce(metadatum_key, default=None), + Coalesce(of_metric(metadatum_key, include_inherited), default=None), key_as_value=True, ), ), @@ -105,12 +129,12 @@ def _spec_actual_per_metric_per_value_dimensions(metadatum_key: str | Spec) -> S ) -def _spec_new_convention(metadatum_key: str | Spec) -> Spec: +def _spec_new_convention(metadatum_key: str | Spec, include_inherited: bool) -> Spec: return ( { # first gather information at different locations in metadata in a dict "metric_key": _spec_metric_key(), "defaults_by_value_dimension": _spec_defaults_by_value_dimension(metadatum_key), - "defaults_by_metric": _spec_defaults_by_metric(metadatum_key), + "defaults_by_metric": _spec_defaults_by_metric(metadatum_key, include_inherited), "actual_per_metric_per_value_dimensions": _spec_actual_per_metric_per_value_dimensions( metadatum_key ), @@ -214,7 +238,7 @@ def _spec_older_convention4(metadatum_key: str | Spec) -> Spec: ) -def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: +def spec_by_metric_key(metadatum_key: str | Spec, include_inherited: bool = True) -> Spec: return Coalesce( ( # current metdadata convention { @@ -224,7 +248,7 @@ def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: Check(instance_of=list), _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( S.globals.metric_key, - Coalesce(metadatum_key, default=None), + Coalesce(of_metric(metadatum_key, include_inherited), default=None), key_as_value=True, ), ), diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index d0ff0c4..dd19299 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -31,12 +31,13 @@ def get_units( - multitsframe: pd.DataFrame, + multitsframe: pd.DataFrame, *, include_inherited: bool = True ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets unit of value dimensions in MTSF metrics from Metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: dict[str, dict[str | None] | None]: Dictionary of metrics containing the names of the value dimensions. @@ -78,14 +79,17 @@ def get_units( """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.UNIT) + return get_value_dimension_info(multitsframe, _specs.UNIT, include_inherited) -def get_names(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdict[str, str | None]]: +def get_names( + multitsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, defaultdict[str, str | None]]: """Gets names of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: dict[str, str | None]: Dictionary of metrics containing the names. If the name is not present for a metric the corresponding value is None. @@ -123,14 +127,17 @@ def get_names(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdict[str, s """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.NAME) + return get_value_dimension_info(multitsframe, _specs.NAME, include_inherited) -def get_display_names(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdict[str, str | None]]: +def get_display_names( + multitsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, defaultdict[str, str | None]]: """Gets display names of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the display names. @@ -169,16 +176,17 @@ def get_display_names(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdic """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.DISPLAY_NAME) + return get_value_dimension_info(multitsframe, _specs.DISPLAY_NAME, include_inherited) def get_short_display_names( - multitsframe: pd.DataFrame, + multitsframe: pd.DataFrame, *, include_inherited: bool = True ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets short display names of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the short display names. @@ -220,14 +228,17 @@ def get_short_display_names( """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.SHORT_DISPLAY_NAME) + return get_value_dimension_info(multitsframe, _specs.SHORT_DISPLAY_NAME, include_inherited) -def get_measurements(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdict[str, str | None]]: +def get_measurements( + multitsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, defaultdict[str, str | None]]: """Gets measurement (type) of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the measurement (type) of the MTSF metrics. @@ -250,10 +261,12 @@ def get_measurements(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdict """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.MEASUREMENT) + return get_value_dimension_info(multitsframe, _specs.MEASUREMENT, include_inherited) -def get_metric_info(multitsframe: pd.DataFrame, metric_info: str | Spec) -> defaultdict[str, Any]: +def get_metric_info( + multitsframe: pd.DataFrame, metric_info: str | Spec, *, include_inherited: bool = True +) -> defaultdict[str, Any]: """Gets a dictionary of metadata associated to metrics. In contrast to metadata associated to concrete value dimensions, this @@ -262,6 +275,7 @@ def get_metric_info(multitsframe: pd.DataFrame, metric_info: str | Spec) -> defa Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata @@ -310,12 +324,14 @@ def get_metric_info(multitsframe: pd.DataFrame, metric_info: str | Spec) -> defa True """ check_dataframe(multitsframe) - spec = spec_by_metric_key(metric_info) + spec = spec_by_metric_key(metric_info, include_inherited) metric_info = glom(multitsframe.attrs, spec) return defaultdict(lambda: None, metric_info) -def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: +def get_series_info( + series: pd.Series, value_dim_info: str | Spec, *, include_inherited: bool = True +) -> Any: """Gets an arbitrary series info Since a series has only one value dimension named "value", this information is @@ -324,6 +340,7 @@ def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: Args: series (pd.Series): Series with metadata following the convention. value_dim_info (str | Spec): Name of information to retrieve. Note that `value_dim_info` is interpreted as a glom Spec. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: Any: Retrieved information defined by `value_dim_info` @@ -345,9 +362,9 @@ def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: # this value dimension. check_series(series) series_metric_key = extract_from_metadata(series.attrs, key="single_metric", default="series") - from_new_convention = get_value_dimension_info(series, value_dim_info)[series_metric_key].get( - "value" - ) + from_new_convention = get_value_dimension_info(series, value_dim_info, include_inherited)[ + series_metric_key + ].get("value") if from_new_convention is not None: return from_new_convention @@ -373,11 +390,12 @@ def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: ) -def get_series_unit(series: pd.Series) -> str | None: +def get_series_unit(series: pd.Series, *, include_inherited: bool = True) -> str | None: """Gets name of the series from metadata Args: series (pd.Series): Series with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: str | None: @@ -415,14 +433,18 @@ def get_series_unit(series: pd.Series) -> str | None: """ check_series(series) - return cast(str | None, get_series_info(series, spec_not_none("unit"))) + return cast( + str | None, + get_series_info(series, spec_not_none("unit"), include_inherited=include_inherited), + ) -def get_series_name(series: pd.Series) -> str | None: +def get_series_name(series: pd.Series, *, include_inherited: bool = True) -> str | None: """Gets name of the series from metadata Args: series (pd.Series): Series with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: str | None: @@ -462,14 +484,18 @@ def get_series_name(series: pd.Series) -> str | None: 'name_of_series_2' """ check_series(series) - return cast(str | None, get_series_info(series, spec_not_none("name"))) + return cast( + str | None, + get_series_info(series, spec_not_none("name"), include_inherited=include_inherited), + ) -def get_series_display_name(series: pd.Series) -> str | None: +def get_series_display_name(series: pd.Series, *, include_inherited: bool = True) -> str | None: """Gets display name of the series from metadata Args: series (pd.Series): Series with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: str | None: @@ -499,15 +525,19 @@ def get_series_display_name(series: pd.Series) -> str | None: spec_not_none("name"), spec_not_none("short_display_name"), ), + include_inherited=include_inherited, ), ) -def get_series_short_display_name(series: pd.Series) -> str | None: +def get_series_short_display_name( + series: pd.Series, *, include_inherited: bool = True +) -> str | None: """Gets short display name of the Series from metadata Args: series (pd.Series): Series with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: str | None: @@ -537,15 +567,17 @@ def get_series_short_display_name(series: pd.Series) -> str | None: spec_not_none("display_name"), spec_not_none("name"), ), + include_inherited=include_inherited, ), ) -def get_series_measurement(series: pd.Series) -> str | None: +def get_series_measurement(series: pd.Series, *, include_inherited: bool = True) -> str | None: """Gets measurement (type) of the Series from metadata Args: series (pd.Series): Series with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: str | None: @@ -565,10 +597,14 @@ def get_series_measurement(series: pd.Series) -> str | None: 'temperature' """ check_series(series) - return cast(str | None, get_series_info(series, "measurement")) + return cast( + str | None, get_series_info(series, "measurement", include_inherited=include_inherited) + ) -def get_singlets_info(singletsframe: pd.DataFrame, value_dim_info: str | Spec) -> Any: +def get_singlets_info( + singletsframe: pd.DataFrame, value_dim_info: str | Spec, *, include_inherited: bool = True +) -> Any: """Gets an arbitrary info of the value dimensions of a SingleTSFrame A SingleTSFrame (STSF) holds exactly one metric - like a Series - but arbitrarily many @@ -582,6 +618,7 @@ def get_singlets_info(singletsframe: pd.DataFrame, value_dim_info: str | Spec) - Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. value_dim_info (str | Spec): Name of information to retrieve. Note that `value_dim_info` is interpreted as a glom Spec. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, Any]: Dictionary of value dimensions containing the retrieved information. If the information is not present for a value dimension the corresponding value is None. @@ -613,14 +650,17 @@ def get_singlets_info(singletsframe: pd.DataFrame, value_dim_info: str | Spec) - """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, value_dim_info) + return singlets_value_dimension_info(singletsframe, value_dim_info, include_inherited) -def get_singlets_units(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: +def get_singlets_units( + singletsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, str | None]: """Gets units of the value dimensions of a SingleTSFrame from metadata Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the units. If the unit is not present for a value dimension the corresponding value is None. @@ -669,14 +709,17 @@ def get_singlets_units(singletsframe: pd.DataFrame) -> defaultdict[str, str | No """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.UNIT) + return singlets_value_dimension_info(singletsframe, _specs.UNIT, include_inherited) -def get_singlets_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: +def get_singlets_names( + singletsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, str | None]: """Gets names of the value dimensions of a SingleTSFrame from metadata Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the names. If the name is not present for a value dimension the corresponding value is None. @@ -708,16 +751,19 @@ def get_singlets_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | No """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.NAME) + return singlets_value_dimension_info(singletsframe, _specs.NAME, include_inherited) -def get_singlets_display_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: +def get_singlets_display_names( + singletsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, str | None]: """Gets display names of the value dimensions of a SingleTSFrame from metadata Falls back to the name of the value dimension if no display name is given. Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the display names. If neither display name nor name is present for a value dimension the corresponding value is None. @@ -749,10 +795,12 @@ def get_singlets_display_names(singletsframe: pd.DataFrame) -> defaultdict[str, """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.DISPLAY_NAME) + return singlets_value_dimension_info(singletsframe, _specs.DISPLAY_NAME, include_inherited) -def get_singlets_short_display_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: +def get_singlets_short_display_names( + singletsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, str | None]: """Gets short display names of the value dimensions of a SingleTSFrame from metadata Falls back to the display name and then to the name of the value dimension if no short @@ -760,6 +808,7 @@ def get_singlets_short_display_names(singletsframe: pd.DataFrame) -> defaultdict Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the short display names. If none of short display name, display name and name is present for a value dimension the corresponding value is None. @@ -791,14 +840,19 @@ def get_singlets_short_display_names(singletsframe: pd.DataFrame) -> defaultdict """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.SHORT_DISPLAY_NAME) + return singlets_value_dimension_info( + singletsframe, _specs.SHORT_DISPLAY_NAME, include_inherited + ) -def get_singlets_measurements(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: +def get_singlets_measurements( + singletsframe: pd.DataFrame, *, include_inherited: bool = True +) -> defaultdict[str, str | None]: """Gets measurements (types) of the value dimensions of a SingleTSFrame from metadata Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the measurements. If "measurement" is not given for a value dimension the corresponding value is None. @@ -826,10 +880,12 @@ def get_singlets_measurements(singletsframe: pd.DataFrame) -> defaultdict[str, s """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.MEASUREMENT) + return singlets_value_dimension_info(singletsframe, _specs.MEASUREMENT, include_inherited) -def get_singlets_metric_info(singletsframe: pd.DataFrame, metric_info: str | Spec) -> Any: +def get_singlets_metric_info( + singletsframe: pd.DataFrame, metric_info: str | Spec, *, include_inherited: bool = True +) -> Any: """Gets metadata associated to the single metric of a SingleTSFrame Counterpart of :func:`get_metric_info` for SingleTSFrames: instead of a dictionary keyed @@ -838,6 +894,7 @@ def get_singlets_metric_info(singletsframe: pd.DataFrame, metric_info: str | Spe Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: Any: Retrieved information defined by `metric_info`, or None if it cannot be determined - which includes the case of absent metric metadata. @@ -868,7 +925,7 @@ def get_singlets_metric_info(singletsframe: pd.DataFrame, metric_info: str | Spe """ check_dataframe(singletsframe) - return singlets_metric_info(singletsframe, metric_info) + return singlets_metric_info(singletsframe, metric_info, include_inherited) def get_queried_interval( diff --git a/tests/metadata/test_metadata.py b/tests/metadata/test_metadata.py index d09a127..e073ae8 100644 --- a/tests/metadata/test_metadata.py +++ b/tests/metadata/test_metadata.py @@ -501,3 +501,89 @@ def test_get_singlets_without_metadata(): def test_get_singlets_requires_a_dataframe(func): with pytest.raises(TypeError): func(pd.Series()) + + +@pytest.fixture +def attrs_with_inherited() -> dict: + return { + "dataset_metadata": {"metric_key": "id", "single_metric": "sensor"}, + "value_dimensions_shared": [{"column": "value", "unit": "shared unit"}], + "metrics": [ + { + "id": "sensor", + "name": "sensor name", + "Latitude": None, + "Longitude": 7.58, + "inherited": { + "Latitude": 51.43, + "Longitude": 8.28, + "display_name": "plant display name", + "unit": "inherited unit", + }, + "value_dimensions": [{"column": "value"}, {"column": "state"}], + }, + {"id": "other", "value_dimensions": [{"column": "value"}]}, + ], + } + + +def test_get_metric_info_falls_back_to_inherited(attrs_with_inherited): + from hdhelpers.metadata import get_metric_info + + dataframe = pd.DataFrame() + dataframe.attrs = attrs_with_inherited + + assert get_metric_info(dataframe, "Latitude")["sensor"] == 51.43 # own None -> inherited + assert get_metric_info(dataframe, "Longitude")["sensor"] == 7.58 # own beats inherited + assert get_metric_info(dataframe, "Latitude")["other"] is None + assert get_metric_info(dataframe, "Latitude", include_inherited=False)["sensor"] is None + assert get_singlets_metric_info(dataframe, "Latitude") == 51.43 + assert get_singlets_metric_info(dataframe, "Latitude", include_inherited=False) is None + + +def test_get_value_dimension_info_falls_back_to_inherited_for_value_only( + attrs_with_inherited, +): + from hdhelpers.metadata import get_units + + dataframe = pd.DataFrame() + dataframe.attrs = attrs_with_inherited + + # like the metric metadata itself, inherited metadata only applies to the "value" column + assert get_singlets_info(dataframe, "Latitude")["value"] == 51.43 + assert get_singlets_info(dataframe, "Latitude")["state"] is None + assert get_singlets_info(dataframe, "Latitude", include_inherited=False)["value"] is None + + # inherited metadata of the metric beats the dataset wide value_dimensions_shared + assert get_units(dataframe)["sensor"]["value"] == "inherited unit" + assert get_units(dataframe)["other"]["value"] == "shared unit" + assert get_units(dataframe, include_inherited=False)["sensor"]["value"] == "shared unit" + assert get_singlets_units(dataframe)["value"] == "inherited unit" + assert get_singlets_units(dataframe, include_inherited=False)["value"] == "shared unit" + + +def test_get_series_info_falls_back_to_inherited(attrs_with_inherited): + from hdhelpers.metadata import get_series_info + + series = pd.Series() + series.attrs = attrs_with_inherited + + assert get_series_info(series, "Latitude") == 51.43 + assert get_series_info(series, "Longitude") == 7.58 + assert get_series_info(series, "Latitude", include_inherited=False) is None + assert get_series_unit(series) == "inherited unit" + assert get_series_unit(series, include_inherited=False) == "shared unit" + + +def test_own_metadata_beats_inherited_metadata_in_fallback_chains(attrs_with_inherited): + """The whole spec is applied to the metric itself before trying its inherited metadata""" + series = pd.Series() + series.attrs = attrs_with_inherited + + # own "name" beats inherited "display_name" + assert get_series_display_name(series) == "sensor name" + assert get_series_short_display_name(series) == "sensor name" + + del series.attrs["metrics"][0]["name"] + assert get_series_display_name(series) == "plant display name" + assert get_series_display_name(series, include_inherited=False) is None From ccb2cb74c3bbd85db664aefaa39217d16e0504c6 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Tue, 20 Oct 2026 22:36:45 +0200 Subject: [PATCH 02/15] fixes around fallback behaviour (null, propagation to weaker fields) --- src/hdhelpers/metadata/_helpers.py | 62 +++++- src/hdhelpers/metadata/_specs.py | 6 + src/hdhelpers/metadata/helpers.py | 111 ++++++++--- tests/metadata/test_metadata_fallbacks.py | 227 ++++++++++++++++++++++ 4 files changed, 371 insertions(+), 35 deletions(-) create mode 100644 tests/metadata/test_metadata_fallbacks.py diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index e03a13b..3d3b386 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -68,15 +68,54 @@ def get_value_dimension_info( ) +def follows_current_convention(attrs: Any) -> bool: + """Whether metadata follows the current metadata convention, i.e. has a list of metrics""" + return isinstance(attrs, dict) and isinstance(attrs.get("metrics"), list) + + +def value_dimension_info_with_fallbacks( + data: pd.DataFrame | pd.Series, + fields: tuple[str, ...], + legacy_spec: Spec, + include_inherited: bool = True, +) -> defaultdict[str, defaultdict[str, Any]]: + """Obtain a field of the value dimensions, falling back to further fields + + For metadata following the current metadata convention, each field is looked up on all + levels described in get_value_dimension_info (the value dimension, the metric for the + value dimension "value" and "value_dimensions_shared") before the next field is tried. + So e.g. a display name of the metric wins over the name of its value dimension. Fields + provided as null are treated as missing. Inherited properties are only used if none of the + fields is provided by the metadata itself. + + Older metadata structures are handled via legacy_spec, keeping their established behaviour. + """ + if not follows_current_convention(data.attrs): + return get_value_dimension_info(data, legacy_spec, include_inherited) + + result: defaultdict[str, defaultdict[str, Any]] = defaultdict(lambda: defaultdict(lambda: None)) + # inherited properties only after all fields of the metadata itself + for inherited in [False, True] if include_inherited else [False]: + for field in fields: + info_by_metric = get_value_dimension_info( + data, Coalesce(field, default=None), inherited + ) + for metric, info_by_value_dimension in info_by_metric.items(): + for value_dimension, info in info_by_value_dimension.items(): + if result[metric][value_dimension] is None: + result[metric][value_dimension] = info + return result + + _SINGLE_METRIC_KEY = "single_metric" def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: - """Pick the entry of the one metric of a SingleTSFrame out of a by-metric mapping + """Pick the entry of the one metric of a SingleTSFrame or Series out of a by-metric mapping The single metric is identified via _SINGLE_METRIC_KEY in "dataset_metadata". If that is - missing but exactly one metric is present, that one is used, since a SingleTSFrame cannot - be ambiguous in this respect. Otherwise `empty` is returned. + missing but exactly one metric is present, that one is used, since a SingleTSFrame or + Series cannot be ambiguous in this respect. Otherwise `empty` is returned. """ metric = extract_from_metadata(attrs, key=_SINGLE_METRIC_KEY) if metric is not None and metric in info_by_metric: @@ -111,6 +150,23 @@ def singlets_value_dimension_info( ) +def singlets_value_dimension_info_with_fallbacks( + singletsframe: pd.DataFrame, + fields: tuple[str, ...], + legacy_spec: Spec, + include_inherited: bool = True, +) -> defaultdict[str, Any]: + """Counterpart of value_dimension_info_with_fallbacks for SingleTSFrames + + Like singlets_value_dimension_info, the result is keyed by value dimension only. + """ + return _select_single_metric( + value_dimension_info_with_fallbacks(singletsframe, fields, legacy_spec, include_inherited), + singletsframe.attrs, + empty=defaultdict(lambda: None), + ) + + def singlets_metric_info( singletsframe: pd.DataFrame, metric_info: str | Spec, include_inherited: bool = True ) -> Any: diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index 71738f9..664c60f 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -16,6 +16,12 @@ # higher up in an asset hierarchy, e.g. the latitude of the plant a sensor belongs to. INHERITED = "inherited" +# The fallback chains of the display names for metadata following the current metadata +# convention: Each field is looked up on all levels (value dimension, metric, shared value +# dimensions) before the next field is tried, see value_dimension_info_with_fallbacks. +DISPLAY_NAME_FIELDS = ("display_name", "name") +SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", "display_name", "name") + def of_metric(metadatum_key: str | Spec, include_inherited: bool = True) -> Spec: """Provides glom spec that extracts a metadatum from a metric object diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index dd19299..32ee99c 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -14,19 +14,22 @@ from typing import Any, cast import pandas as pd -from glom import Coalesce, Spec, glom +from glom import Coalesce, GlomError, Spec, glom +import hdhelpers.metadata._specs as _specs from hdhelpers.metadata._helpers import ( check_dataframe, check_series, check_series_or_dataframe, extract_from_metadata, + follows_current_convention, get_value_dimension_info, singlets_metric_info, singlets_value_dimension_info, + singlets_value_dimension_info_with_fallbacks, spec_not_none, + value_dimension_info_with_fallbacks, ) -import hdhelpers.metadata._specs as _specs from hdhelpers.metadata._specs import spec_by_metric_key @@ -142,6 +145,9 @@ def get_display_names( Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the display names. If the display name of the metrics is not present it returns the result of :func:`hdhelpers.metadata.get_names`. + For metadata following the current convention, the display name is looked up on the value dimension, on + the metric (for the value dimension "value") and in "value_dimensions_shared" before falling back to the + name. Fields provided as null count as missing. Raises: TypeError: If `multitsframe` is not a DataFrame. @@ -176,7 +182,9 @@ def get_display_names( """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.DISPLAY_NAME, include_inherited) + return value_dimension_info_with_fallbacks( + multitsframe, _specs.DISPLAY_NAME_FIELDS, _specs.DISPLAY_NAME, include_inherited + ) def get_short_display_names( @@ -191,6 +199,9 @@ def get_short_display_names( Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the short display names. If the short display name of the metrics is not present it returns the result of :func:`hdhelpers.metadata.get_display_names`. + For metadata following the current convention, each of short display name, display name and name is + looked up on the value dimension, on the metric (for the value dimension "value") and in + "value_dimensions_shared" before falling back to the next one. Fields provided as null count as missing. Raises: TypeError: If `multitsframe` is not a DataFrame. @@ -228,7 +239,9 @@ def get_short_display_names( """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.SHORT_DISPLAY_NAME, include_inherited) + return value_dimension_info_with_fallbacks( + multitsframe, _specs.SHORT_DISPLAY_NAME_FIELDS, _specs.SHORT_DISPLAY_NAME, include_inherited + ) def get_measurements( @@ -278,7 +291,7 @@ def get_metric_info( include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. Returns: - defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata + defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata. Defaults to None, also if there is no metric metadata at all. Code example: @@ -325,8 +338,11 @@ def get_metric_info( """ check_dataframe(multitsframe) spec = spec_by_metric_key(metric_info, include_inherited) - metric_info = glom(multitsframe.attrs, spec) - return defaultdict(lambda: None, metric_info) + try: + info_by_metric = glom(multitsframe.attrs, spec) + except GlomError: # no metric metadata present at all + info_by_metric = {} + return defaultdict(lambda: None, info_by_metric) def get_series_info( @@ -361,10 +377,9 @@ def get_series_info( # metadata, we can reuse the code that extracts value_dimension metadata for # this value dimension. check_series(series) - series_metric_key = extract_from_metadata(series.attrs, key="single_metric", default="series") - from_new_convention = get_value_dimension_info(series, value_dim_info, include_inherited)[ - series_metric_key - ].get("value") + from_new_convention = _series_value( + get_value_dimension_info(series, value_dim_info, include_inherited), series + ) if from_new_convention is not None: return from_new_convention @@ -390,6 +405,40 @@ def get_series_info( ) +def _series_value(info_by_metric: dict, series: pd.Series) -> Any: + """Info of the value dimension "value" of the metric of a Series + + The metric is identified via "single_metric" in "dataset_metadata", defaulting to "series" + as used by older metadata structures. If it is not found but exactly one metric is present, + that one is used, since a Series cannot be ambiguous in this respect. + """ + series_metric_key = extract_from_metadata(series.attrs, key="single_metric", default="series") + if series_metric_key in info_by_metric: + return info_by_metric[series_metric_key].get("value") + if len(info_by_metric) == 1: + return next(iter(info_by_metric.values())).get("value") + return None + + +def _series_info_with_fallbacks( + series: pd.Series, fields: tuple[str, ...], include_inherited: bool +) -> Any: + """Info of a Series, falling back to further fields + + For metadata following the current metadata convention, each field is looked up on the + value dimension and the metric before the next field is tried, see + value_dimension_info_with_fallbacks. Older metadata structures keep their established + behaviour of using the first of the fields present on the first level providing any. + """ + legacy_spec = Coalesce(*(spec_not_none(field) for field in fields)) + if follows_current_convention(series.attrs): + return _series_value( + value_dimension_info_with_fallbacks(series, fields, legacy_spec, include_inherited), + series, + ) + return get_series_info(series, legacy_spec, include_inherited=include_inherited) + + def get_series_unit(series: pd.Series, *, include_inherited: bool = True) -> str | None: """Gets name of the series from metadata @@ -518,14 +567,8 @@ def get_series_display_name(series: pd.Series, *, include_inherited: bool = True check_series(series) return cast( str | None, - get_series_info( - series, - Coalesce( - spec_not_none("display_name"), - spec_not_none("name"), - spec_not_none("short_display_name"), - ), - include_inherited=include_inherited, + _series_info_with_fallbacks( + series, ("display_name", "name", "short_display_name"), include_inherited ), ) @@ -560,14 +603,8 @@ def get_series_short_display_name( check_series(series) return cast( str | None, - get_series_info( - series, - Coalesce( - spec_not_none("short_display_name"), - spec_not_none("display_name"), - spec_not_none("name"), - ), - include_inherited=include_inherited, + _series_info_with_fallbacks( + series, ("short_display_name", "display_name", "name"), include_inherited ), ) @@ -759,7 +796,10 @@ def get_singlets_display_names( ) -> defaultdict[str, str | None]: """Gets display names of the value dimensions of a SingleTSFrame from metadata - Falls back to the name of the value dimension if no display name is given. + Falls back to the name of the value dimension if no display name is given. The display name + is looked up on the value dimension, on the metric (for the value dimension "value") and in + "value_dimensions_shared" before falling back to the name. Fields provided as null count as + missing. Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. @@ -795,7 +835,9 @@ def get_singlets_display_names( """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.DISPLAY_NAME, include_inherited) + return singlets_value_dimension_info_with_fallbacks( + singletsframe, _specs.DISPLAY_NAME_FIELDS, _specs.DISPLAY_NAME, include_inherited + ) def get_singlets_short_display_names( @@ -804,7 +846,9 @@ def get_singlets_short_display_names( """Gets short display names of the value dimensions of a SingleTSFrame from metadata Falls back to the display name and then to the name of the value dimension if no short - display name is given. + display name is given. Each of them is looked up on the value dimension, on the metric (for + the value dimension "value") and in "value_dimensions_shared" before falling back to the next + one. Fields provided as null count as missing. Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. @@ -840,8 +884,11 @@ def get_singlets_short_display_names( """ check_dataframe(singletsframe) - return singlets_value_dimension_info( - singletsframe, _specs.SHORT_DISPLAY_NAME, include_inherited + return singlets_value_dimension_info_with_fallbacks( + singletsframe, + _specs.SHORT_DISPLAY_NAME_FIELDS, + _specs.SHORT_DISPLAY_NAME, + include_inherited, ) diff --git a/tests/metadata/test_metadata_fallbacks.py b/tests/metadata/test_metadata_fallbacks.py new file mode 100644 index 0000000..05e7023 --- /dev/null +++ b/tests/metadata/test_metadata_fallbacks.py @@ -0,0 +1,227 @@ +"""Fallbacks for metadata following the current metadata convention + +Metadata providers like the hetida platform send fields without value explicitly as null. +These count as missing: The accessors fall back to the next level or field as if the field +was not provided at all. +""" + +from copy import deepcopy + +import pandas as pd +import pytest + +from hdhelpers.metadata import ( + get_display_names, + get_measurements, + get_metric_info, + get_names, + get_series_display_name, + get_series_measurement, + get_series_name, + get_series_short_display_name, + get_series_unit, + get_short_display_names, + get_singlets_display_names, + get_singlets_short_display_names, + get_singlets_units, + get_units, +) + +SIGNAL_ID = "7ab8c419-8745-4714-b588-57c0c772cc95" + + +def value_dimension(**fields): + """Value dimension "value" as the hetida platform sends it, with nulls for missing fields""" + return { + "column": "value", + "name": None, + "display_name": None, + "short_display_name": None, + "description": None, + "unit": None, + "measurement": None, + "value_data_type": "float", + } | fields + + +def metric(**fields): + """Metric as the hetida platform sends it, with nulls for missing fields""" + return { + "id": SIGNAL_ID, + "external_id": None, + "name": None, + "display_name": None, + "description": None, + "inherited": {}, + "comments": [], + "value_dimensions": [value_dimension()], + "default_aggregation_method": None, + } | fields + + +# what the hetida platform sends for a timeseries of a SERIES read +platform_series_attrs = { + "dataset_metadata": { + "metric_key": "id", + "single_metric": SIGNAL_ID, + "queried_metrics": [SIGNAL_ID], + "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", + "ref_interval_end_timestamp": "2026-10-19T00:00:00Z", + "ref_interval_type": "closed", + }, + "metrics": [ + metric( + external_id="water.north.level", + name="Water-lvl-north", + value_dimensions=[ + value_dimension(name="Water-lvl-north", unit="m", measurement="level") + ], + default_aggregation_method="AVG", + ) + ], +} + +# what the hetida platform sends for a MULTITSFRAME read of a signal group: display_name of +# a metric is the display name configured for the member of the group +platform_multitsframe_attrs = { + "dataset_metadata": {"metric_key": "id", "queried_metrics": ["level", "flow"]}, + "metrics": [ + metric( + id="level", + name="Water-lvl-north", + value_dimensions=[value_dimension(name="Water-lvl-north", unit="m")], + ), + metric( + id="flow", + name="Pump-flow", + display_name="Flow pump 1", + value_dimensions=[value_dimension(name="Pump-flow")], + ), + ], +} + + +def series(attrs): + ser = pd.Series([1.0], index=pd.to_datetime(["2026-10-18T06:00:00Z"])) + ser.attrs = deepcopy(attrs) + return ser + + +def frame(attrs): + df = pd.DataFrame() + df.attrs = deepcopy(attrs) + return df + + +def test_series_from_platform(): + platform_series = series(platform_series_attrs) + + assert get_series_unit(platform_series) == "m" + assert get_series_measurement(platform_series) == "level" + assert get_series_name(platform_series) == "Water-lvl-north" + assert get_series_display_name(platform_series) == "Water-lvl-north" + assert get_series_short_display_name(platform_series) == "Water-lvl-north" + + +def test_multitsframe_from_platform(): + multitsframe = frame(platform_multitsframe_attrs) + + assert get_units(multitsframe)["level"]["value"] == "m" + assert get_units(multitsframe)["flow"]["value"] is None + assert get_names(multitsframe)["flow"]["value"] == "Pump-flow" + assert get_measurements(multitsframe)["level"]["value"] is None + # a null display name falls back to the name + assert get_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" + assert get_short_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" + # the display name of the metric wins over the name of its value dimension + assert get_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" + assert get_short_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" + + +def test_each_field_is_looked_up_on_all_levels_before_the_next_field(): + attrs = { + "metrics": [ + metric( + id="a", + name="metric name", + value_dimensions=[ + value_dimension(name="value name"), + value_dimension(column="state", name="state name"), + ], + ), + metric(id="b", name="metric name", display_name="metric display name"), + metric( + id="c", + display_name="metric display name", + value_dimensions=[value_dimension(display_name="value display name")], + ), + ], + "value_dimensions_shared": [{"column": "state", "display_name": "shared display name"}], + } + + display_names = get_display_names(frame(attrs)) + + assert display_names["a"]["value"] == "value name" + assert display_names["a"]["state"] == "shared display name" + assert display_names["b"]["value"] == "metric display name" + assert display_names["c"]["value"] == "value display name" + + +def test_singletsframe_display_names(): + attrs = { + "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, + "metrics": [ + metric( + external_id="abc.temp", + name="ABC temperature", + display_name="ABC temp.", + value_dimensions=[ + value_dimension(name="temperature", unit="°C"), + value_dimension(column="state", name="measurement state"), + ], + ) + ], + } + singletsframe = frame(attrs) + + assert get_singlets_units(singletsframe)["value"] == "°C" + # only the value dimension "value" falls back to the metric + assert get_singlets_display_names(singletsframe)["value"] == "ABC temp." + assert get_singlets_display_names(singletsframe)["state"] == "measurement state" + assert get_singlets_short_display_names(singletsframe)["state"] == "measurement state" + + +def test_series_display_name_of_metric_wins_over_value_dimension_name(): + attrs = deepcopy(platform_series_attrs) + attrs["metrics"][0]["display_name"] = "Level north" + attrs["metrics"][0]["short_display_name"] = "north" + + assert get_series_display_name(series(attrs)) == "Level north" + assert get_series_short_display_name(series(attrs)) == "north" + + +def test_series_without_single_metric_uses_its_only_metric(): + attrs = deepcopy(platform_series_attrs) + del attrs["dataset_metadata"]["single_metric"] + + assert get_series_unit(series(attrs)) == "m" + assert get_series_display_name(series(attrs)) == "Water-lvl-north" + + +@pytest.mark.parametrize( + "attrs", + [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}], +) +def test_metric_info_without_metric_metadata(attrs): + """Components should not require metadata, so missing metric metadata must not raise""" + assert get_metric_info(frame(attrs), "name")["any metric"] is None + + +def test_metric_info_from_platform(): + multitsframe = frame(platform_multitsframe_attrs) + multitsframe.attrs["metrics"][0]["default_aggregation_method"] = "SUM" + + aggregation_methods = get_metric_info(multitsframe, "default_aggregation_method") + + assert aggregation_methods["level"] == "SUM" + assert aggregation_methods["flow"] is None From 87047ab1cba2783d9e028a31560423871d7330f5 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Wed, 21 Oct 2026 09:37:53 +0200 Subject: [PATCH 03/15] switch to more sensible default and fallback behaviours --- src/hdhelpers/metadata/_helpers.py | 54 +++++++++---- src/hdhelpers/metadata/_specs.py | 126 +++++++++++++++++------------ src/hdhelpers/metadata/helpers.py | 93 +++++++++++---------- tests/metadata/test_metadata.py | 60 +++++++++++--- 4 files changed, 212 insertions(+), 121 deletions(-) diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 3d3b386..f9c528b 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -35,7 +35,7 @@ def extract_from_metadata(metadata: Any, key: str, default: str | None = None) - def get_value_dimension_info( multitsframe: pd.DataFrame | pd.Series, value_dim_info: str | Spec, - include_inherited: bool = True, + include_inherited: bool = False, ) -> defaultdict[str, defaultdict[str, Any]]: """Obtain metadata info associated to the value dimensions of the metrics @@ -53,8 +53,8 @@ def get_value_dimension_info( dimension in the value_dimensions list under the metric, the global "value_dimensions_shared" field of the attrs object is searched for corresponding information. - If include_inherited, the properties a metric inherits are used as part of its metric - metadata, i.e. after the metric's own information but before "value_dimensions_shared". + If include_inherited, the properties a metric inherits are used as last resort for the + "value" value dimension, i.e. after "value_dimensions_shared". If no information is found, None is set as value and is the default value of the inner default dict. @@ -77,7 +77,7 @@ def value_dimension_info_with_fallbacks( data: pd.DataFrame | pd.Series, fields: tuple[str, ...], legacy_spec: Spec, - include_inherited: bool = True, + include_inherited: bool = False, ) -> defaultdict[str, defaultdict[str, Any]]: """Obtain a field of the value dimensions, falling back to further fields @@ -128,7 +128,7 @@ def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: def singlets_value_dimension_info( - singletsframe: pd.DataFrame, value_dim_info: str | Spec, include_inherited: bool = True + singletsframe: pd.DataFrame, value_dim_info: str | Spec, include_inherited: bool = False ) -> defaultdict[str, Any]: """Metadata info of the value dimensions of the single metric of a SingleTSFrame @@ -154,7 +154,7 @@ def singlets_value_dimension_info_with_fallbacks( singletsframe: pd.DataFrame, fields: tuple[str, ...], legacy_spec: Spec, - include_inherited: bool = True, + include_inherited: bool = False, ) -> defaultdict[str, Any]: """Counterpart of value_dimension_info_with_fallbacks for SingleTSFrames @@ -167,6 +167,35 @@ def singlets_value_dimension_info_with_fallbacks( ) +def metric_info_by_metric( + data: pd.DataFrame | pd.Series, metric_info: str | Spec, include_inherited: bool = True +) -> dict[str, Any]: + """Obtain metadata info associated to the metrics, keyed by metric + + For metadata following the current metadata convention, information the metric itself does + not provide is taken from its value dimension "value", then from "value_dimensions_shared" + and, if include_inherited, finally from the properties the metric inherits. Information + provided as null counts as missing. + + Returns an empty dict if no metric metadata is present at all. + """ + try: + info_by_metric = glom(data.attrs, _specs.spec_by_metric_key(metric_info)) + except GlomError: # no metric metadata present at all + return {} + + if not follows_current_convention(data.attrs): + return info_by_metric + + value_dimension_info = get_value_dimension_info( + data, Coalesce(metric_info, default=None), include_inherited + ) + return { + metric: info if info is not None else value_dimension_info[metric].get("value") + for metric, info in info_by_metric.items() + } + + def singlets_metric_info( singletsframe: pd.DataFrame, metric_info: str | Spec, include_inherited: bool = True ) -> Any: @@ -177,11 +206,8 @@ def singlets_metric_info( be determined - which includes the case of absent metric metadata, since components should not require metadata (see the metadata conventions documentation). """ - try: - info_by_metric = glom( - singletsframe.attrs, _specs.spec_by_metric_key(metric_info, include_inherited) - ) - except GlomError: # no metric metadata present at all - return None - - return _select_single_metric(info_by_metric, singletsframe.attrs, empty=None) + return _select_single_metric( + metric_info_by_metric(singletsframe, metric_info, include_inherited), + singletsframe.attrs, + empty=None, + ) diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index 664c60f..c684709 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -1,7 +1,8 @@ from collections import defaultdict from collections.abc import Callable +from typing import Any -from glom import A, Check, Coalesce, Iter, Merge, S, Spec, T +from glom import A, Check, Coalesce, Iter, Merge, S, Spec, T, Val # The metadatum specs of the individual accessors, including their fallback chains. # Defined once here and shared by the MultiTSFrame and the SingleTSFrame accessors, which @@ -23,23 +24,8 @@ SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", "display_name", "name") -def of_metric(metadatum_key: str | Spec, include_inherited: bool = True) -> Spec: - """Provides glom spec that extracts a metadatum from a metric object - - If include_inherited, the metric's inherited properties are used as fallback: the whole - metadatum_key spec is first applied to the metric itself and only if that yields None - or fails, it is applied to the INHERITED dict of the metric. So in a fallback chain - like DISPLAY_NAME an own "name" beats an inherited "display_name". - - metadatum_key can also be any glom spec. - """ - if not include_inherited: - return metadatum_key - return Coalesce(metadatum_key, (INHERITED, metadatum_key), skip=lambda x: x is None) - - def by_metric_key_by_val_dimension( - metadatum_key: str | Spec, include_inherited: bool = True + metadatum_key: str | Spec, include_inherited: bool = False ) -> Spec: """Providesglom spec that extracts a metadatum by metric by value dimension @@ -54,8 +40,9 @@ def by_metric_key_by_val_dimension( Properly falls back to "value_dimensions_shared" metadata if a value_dimension is not given for a metric if its available there. - For the current metadata convention, properties the metric inherits are used as part of - the metric metadatum if include_inherited, see of_metric. + For the current metadata convention and include_inherited, the properties the metric + inherits are used as last resort for the "value" value_dimension, i.e. after + "value_dimensions_shared". metadatum_key can also be any glom spec. """ @@ -95,14 +82,30 @@ def _spec_defaults_by_value_dimension(metadatum_key: str | Spec) -> Spec: ) -def _spec_defaults_by_metric(metadatum_key: str | Spec, include_inherited: bool) -> Spec: +def _spec_defaults_by_metric(metadatum_key: str | Spec) -> Spec: return Coalesce( ( "metrics", Check(instance_of=list), _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( S.globals.metric_key, - Coalesce(of_metric(metadatum_key, include_inherited), default=None), + Coalesce(metadatum_key, default=None), + key_as_value=True, + ), + ), + default={}, + ) + + +def _spec_inherited_by_metric(metadatum_key: str | Spec) -> Spec: + """The metadatum from the properties each metric inherits, keyed by metric""" + return Coalesce( + ( + "metrics", + Check(instance_of=list), + _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( + S.globals.metric_key, + Coalesce((INHERITED, metadatum_key), default=None), key_as_value=True, ), ), @@ -140,39 +143,57 @@ def _spec_new_convention(metadatum_key: str | Spec, include_inherited: bool) -> { # first gather information at different locations in metadata in a dict "metric_key": _spec_metric_key(), "defaults_by_value_dimension": _spec_defaults_by_value_dimension(metadatum_key), - "defaults_by_metric": _spec_defaults_by_metric(metadatum_key, include_inherited), + "defaults_by_metric": _spec_defaults_by_metric(metadatum_key), + "inherited_by_metric": ( + _spec_inherited_by_metric(metadatum_key) if include_inherited else Val({}) + ), "actual_per_metric_per_value_dimensions": _spec_actual_per_metric_per_value_dimensions( metadatum_key ), }, - lambda x: defaultdict( - lambda: defaultdict(lambda: None, {}), - { # combine dicts with gathered information / falling back to defaults - metric: defaultdict( - lambda: None, - { - value_dim: info - if info is not None # prio 1: use, if explicitely provided - else ( - x["defaults_by_metric"][metric] - if ( - value_dim == "value" - and x["defaults_by_metric"].get(metric) is not None - ) # prio 2: only for "value" value dim: possibly use from metric - else ( - x["defaults_by_value_dimension"].get( - value_dim, None - ) # prio 3: from global "value_dimensions_shared" - ) - ) - for value_dim, info in _update_dict_and_return_it( - x["defaults_by_value_dimension"].copy(), info_by_val_dim - ).items() - }, - ) - for metric, info_by_val_dim in x["actual_per_metric_per_value_dimensions"].items() - }, - ), + _combine_gathered_information, + ) + + +def _first_not_none(*values: Any) -> Any: + return next((value for value in values if value is not None), None) + + +def _combine_gathered_information(gathered: dict) -> defaultdict: + """Combine the information gathered by _spec_new_convention + + For each value dimension of each metric, the first information which is not None is used, + in the following order: + 1. the value dimension itself + 2. only for the value dimension "value": the metric + 3. "value_dimensions_shared" + 4. only for the value dimension "value": the properties the metric inherits + """ + return defaultdict( + lambda: defaultdict(lambda: None, {}), + { + metric: defaultdict( + lambda: None, + { + value_dim: _first_not_none( + info, + gathered["defaults_by_metric"].get(metric) + if value_dim == "value" + else None, + gathered["defaults_by_value_dimension"].get(value_dim), + gathered["inherited_by_metric"].get(metric) + if value_dim == "value" + else None, + ) + for value_dim, info in _update_dict_and_return_it( + gathered["defaults_by_value_dimension"].copy(), info_by_val_dim + ).items() + }, + ) + for metric, info_by_val_dim in gathered[ + "actual_per_metric_per_value_dimensions" + ].items() + }, ) @@ -244,7 +265,8 @@ def _spec_older_convention4(metadatum_key: str | Spec) -> Spec: ) -def spec_by_metric_key(metadatum_key: str | Spec, include_inherited: bool = True) -> Spec: +def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: + """Provides glom spec that extracts a metadatum from the metric objects, keyed by metric""" return Coalesce( ( # current metdadata convention { @@ -254,7 +276,7 @@ def spec_by_metric_key(metadatum_key: str | Spec, include_inherited: bool = True Check(instance_of=list), _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( S.globals.metric_key, - Coalesce(of_metric(metadatum_key, include_inherited), default=None), + Coalesce(metadatum_key, default=None), key_as_value=True, ), ), diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 32ee99c..0174f8b 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -14,7 +14,7 @@ from typing import Any, cast import pandas as pd -from glom import Coalesce, GlomError, Spec, glom +from glom import Coalesce, Spec, glom import hdhelpers.metadata._specs as _specs from hdhelpers.metadata._helpers import ( @@ -24,23 +24,23 @@ extract_from_metadata, follows_current_convention, get_value_dimension_info, + metric_info_by_metric, singlets_metric_info, singlets_value_dimension_info, singlets_value_dimension_info_with_fallbacks, spec_not_none, value_dimension_info_with_fallbacks, ) -from hdhelpers.metadata._specs import spec_by_metric_key def get_units( - multitsframe: pd.DataFrame, *, include_inherited: bool = True + multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets unit of value dimensions in MTSF metrics from Metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: dict[str, dict[str | None] | None]: Dictionary of metrics containing the names of the value dimensions. @@ -86,13 +86,13 @@ def get_units( def get_names( - multitsframe: pd.DataFrame, *, include_inherited: bool = True + multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets names of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: dict[str, str | None]: Dictionary of metrics containing the names. If the name is not present for a metric the corresponding value is None. @@ -134,13 +134,13 @@ def get_names( def get_display_names( - multitsframe: pd.DataFrame, *, include_inherited: bool = True + multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets display names of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the display names. @@ -188,13 +188,13 @@ def get_display_names( def get_short_display_names( - multitsframe: pd.DataFrame, *, include_inherited: bool = True + multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets short display names of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the short display names. @@ -245,13 +245,13 @@ def get_short_display_names( def get_measurements( - multitsframe: pd.DataFrame, *, include_inherited: bool = True + multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: """Gets measurement (type) of the MTSF metrics from the metadata Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the measurement (type) of the MTSF metrics. @@ -285,10 +285,18 @@ def get_metric_info( In contrast to metadata associated to concrete value dimensions, this function abstracts access to metadata associated to the underlying metric. + The information is looked up in the metric itself, then in its value dimension "value", + then in "value_dimensions_shared" and finally in the properties the metric inherits from + objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), + e.g. the latitude of the plant a sensor belongs to. The latter can be switched off via + `include_inherited`. Information provided as null counts as missing. Note that in contrast + to this function, the functions for dedicated information like :func:`get_units` or + :func:`get_display_names` do not fall back to inherited properties by default. + Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the metric, its value dimension "value" nor "value_dimensions_shared" provide the information. Defaults to True. Returns: defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata. Defaults to None, also if there is no metric metadata at all. @@ -337,12 +345,9 @@ def get_metric_info( True """ check_dataframe(multitsframe) - spec = spec_by_metric_key(metric_info, include_inherited) - try: - info_by_metric = glom(multitsframe.attrs, spec) - except GlomError: # no metric metadata present at all - info_by_metric = {} - return defaultdict(lambda: None, info_by_metric) + return defaultdict( + lambda: None, metric_info_by_metric(multitsframe, metric_info, include_inherited) + ) def get_series_info( @@ -356,7 +361,7 @@ def get_series_info( Args: series (pd.Series): Series with metadata following the convention. value_dim_info (str | Spec): Name of information to retrieve. Note that `value_dim_info` is interpreted as a glom Spec. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to True. Returns: Any: Retrieved information defined by `value_dim_info` @@ -439,12 +444,12 @@ def _series_info_with_fallbacks( return get_series_info(series, legacy_spec, include_inherited=include_inherited) -def get_series_unit(series: pd.Series, *, include_inherited: bool = True) -> str | None: +def get_series_unit(series: pd.Series, *, include_inherited: bool = False) -> str | None: """Gets name of the series from metadata Args: series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: str | None: @@ -488,12 +493,12 @@ def get_series_unit(series: pd.Series, *, include_inherited: bool = True) -> str ) -def get_series_name(series: pd.Series, *, include_inherited: bool = True) -> str | None: +def get_series_name(series: pd.Series, *, include_inherited: bool = False) -> str | None: """Gets name of the series from metadata Args: series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: str | None: @@ -539,12 +544,12 @@ def get_series_name(series: pd.Series, *, include_inherited: bool = True) -> str ) -def get_series_display_name(series: pd.Series, *, include_inherited: bool = True) -> str | None: +def get_series_display_name(series: pd.Series, *, include_inherited: bool = False) -> str | None: """Gets display name of the series from metadata Args: series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: str | None: @@ -574,13 +579,13 @@ def get_series_display_name(series: pd.Series, *, include_inherited: bool = True def get_series_short_display_name( - series: pd.Series, *, include_inherited: bool = True + series: pd.Series, *, include_inherited: bool = False ) -> str | None: """Gets short display name of the Series from metadata Args: series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: str | None: @@ -609,12 +614,12 @@ def get_series_short_display_name( ) -def get_series_measurement(series: pd.Series, *, include_inherited: bool = True) -> str | None: +def get_series_measurement(series: pd.Series, *, include_inherited: bool = False) -> str | None: """Gets measurement (type) of the Series from metadata Args: series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: str | None: @@ -655,7 +660,7 @@ def get_singlets_info( Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. value_dim_info (str | Spec): Name of information to retrieve. Note that `value_dim_info` is interpreted as a glom Spec. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to True. Returns: defaultdict[str, Any]: Dictionary of value dimensions containing the retrieved information. If the information is not present for a value dimension the corresponding value is None. @@ -691,13 +696,13 @@ def get_singlets_info( def get_singlets_units( - singletsframe: pd.DataFrame, *, include_inherited: bool = True + singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: """Gets units of the value dimensions of a SingleTSFrame from metadata Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the units. If the unit is not present for a value dimension the corresponding value is None. @@ -750,13 +755,13 @@ def get_singlets_units( def get_singlets_names( - singletsframe: pd.DataFrame, *, include_inherited: bool = True + singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: """Gets names of the value dimensions of a SingleTSFrame from metadata Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the names. If the name is not present for a value dimension the corresponding value is None. @@ -792,7 +797,7 @@ def get_singlets_names( def get_singlets_display_names( - singletsframe: pd.DataFrame, *, include_inherited: bool = True + singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: """Gets display names of the value dimensions of a SingleTSFrame from metadata @@ -803,7 +808,7 @@ def get_singlets_display_names( Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the display names. If neither display name nor name is present for a value dimension the corresponding value is None. @@ -841,7 +846,7 @@ def get_singlets_display_names( def get_singlets_short_display_names( - singletsframe: pd.DataFrame, *, include_inherited: bool = True + singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: """Gets short display names of the value dimensions of a SingleTSFrame from metadata @@ -852,7 +857,7 @@ def get_singlets_short_display_names( Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the short display names. If none of short display name, display name and name is present for a value dimension the corresponding value is None. @@ -893,13 +898,13 @@ def get_singlets_short_display_names( def get_singlets_measurements( - singletsframe: pd.DataFrame, *, include_inherited: bool = True + singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: """Gets measurements (types) of the value dimensions of a SingleTSFrame from metadata Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. Returns: defaultdict[str, str | None]: Dictionary of value dimensions containing the measurements. If "measurement" is not given for a value dimension the corresponding value is None. @@ -936,12 +941,14 @@ def get_singlets_metric_info( """Gets metadata associated to the single metric of a SingleTSFrame Counterpart of :func:`get_metric_info` for SingleTSFrames: instead of a dictionary keyed - by metric this directly returns the requested information for the one metric. + by metric this directly returns the requested information for the one metric. Like + :func:`get_metric_info`, it falls back to the value dimension "value", to + "value_dimensions_shared" and finally to the properties the metric inherits. Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), if the metric itself does not provide the information. Defaults to True. + include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the metric, its value dimension "value" nor "value_dimensions_shared" provide the information. Defaults to True. Returns: Any: Retrieved information defined by `metric_info`, or None if it cannot be determined - which includes the case of absent metric metadata. diff --git a/tests/metadata/test_metadata.py b/tests/metadata/test_metadata.py index e073ae8..4e9db3d 100644 --- a/tests/metadata/test_metadata.py +++ b/tests/metadata/test_metadata.py @@ -554,12 +554,18 @@ def test_get_value_dimension_info_falls_back_to_inherited_for_value_only( assert get_singlets_info(dataframe, "Latitude")["state"] is None assert get_singlets_info(dataframe, "Latitude", include_inherited=False)["value"] is None - # inherited metadata of the metric beats the dataset wide value_dimensions_shared - assert get_units(dataframe)["sensor"]["value"] == "inherited unit" - assert get_units(dataframe)["other"]["value"] == "shared unit" - assert get_units(dataframe, include_inherited=False)["sensor"]["value"] == "shared unit" - assert get_singlets_units(dataframe)["value"] == "inherited unit" - assert get_singlets_units(dataframe, include_inherited=False)["value"] == "shared unit" + # inherited metadata is the last resort, after the dataset wide value_dimensions_shared + assert get_units(dataframe, include_inherited=True)["sensor"]["value"] == "shared unit" + assert get_units(dataframe, include_inherited=True)["other"]["value"] == "shared unit" + assert get_singlets_units(dataframe, include_inherited=True)["value"] == "shared unit" + + del dataframe.attrs["value_dimensions_shared"] + assert get_units(dataframe, include_inherited=True)["sensor"]["value"] == "inherited unit" + assert get_singlets_units(dataframe, include_inherited=True)["value"] == "inherited unit" + + # the functions for dedicated information do not fall back to inherited by default + assert get_units(dataframe)["sensor"]["value"] is None + assert get_singlets_units(dataframe)["value"] is None def test_get_series_info_falls_back_to_inherited(attrs_with_inherited): @@ -571,8 +577,12 @@ def test_get_series_info_falls_back_to_inherited(attrs_with_inherited): assert get_series_info(series, "Latitude") == 51.43 assert get_series_info(series, "Longitude") == 7.58 assert get_series_info(series, "Latitude", include_inherited=False) is None - assert get_series_unit(series) == "inherited unit" - assert get_series_unit(series, include_inherited=False) == "shared unit" + assert get_series_unit(series) == "shared unit" + assert get_series_unit(series, include_inherited=True) == "shared unit" + + del series.attrs["value_dimensions_shared"] + assert get_series_unit(series) is None + assert get_series_unit(series, include_inherited=True) == "inherited unit" def test_own_metadata_beats_inherited_metadata_in_fallback_chains(attrs_with_inherited): @@ -581,9 +591,35 @@ def test_own_metadata_beats_inherited_metadata_in_fallback_chains(attrs_with_inh series.attrs = attrs_with_inherited # own "name" beats inherited "display_name" - assert get_series_display_name(series) == "sensor name" - assert get_series_short_display_name(series) == "sensor name" + assert get_series_display_name(series, include_inherited=True) == "sensor name" + assert get_series_short_display_name(series, include_inherited=True) == "sensor name" del series.attrs["metrics"][0]["name"] - assert get_series_display_name(series) == "plant display name" - assert get_series_display_name(series, include_inherited=False) is None + assert get_series_display_name(series, include_inherited=True) == "plant display name" + assert get_series_display_name(series) is None + + +def test_get_metric_info_prefers_value_dimension_and_shared_over_inherited( + attrs_with_inherited, +): + """Order of get_metric_info: metric, value dimension "value", shared, inherited""" + from hdhelpers.metadata import get_metric_info, get_units + + dataframe = pd.DataFrame() + dataframe.attrs = attrs_with_inherited + + assert get_metric_info(dataframe, "unit")["sensor"] == "shared unit" + + dataframe.attrs["metrics"][0]["value_dimensions"][0]["unit"] = "m" + assert get_metric_info(dataframe, "unit")["sensor"] == "m" + assert get_units(dataframe)["sensor"]["value"] == "m" + + dataframe.attrs["metrics"][0]["unit"] = "kg" + assert get_metric_info(dataframe, "unit")["sensor"] == "kg" + + del dataframe.attrs["value_dimensions_shared"] + del dataframe.attrs["metrics"][0]["unit"] + dataframe.attrs["metrics"][0]["value_dimensions"][0]["unit"] = None + assert get_metric_info(dataframe, "unit")["sensor"] == "inherited unit" + assert get_metric_info(dataframe, "unit", include_inherited=False)["sensor"] is None + assert get_singlets_metric_info(dataframe, "unit") == "inherited unit" From 28e729b42535ed7dd871ef1cc4ae457184401d69 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Wed, 21 Oct 2026 11:11:34 +0200 Subject: [PATCH 04/15] changelog and documentation changes --- CHANGELOG.md | 6 +- docs/_sources/first_steps.rst.txt | 79 +++++++++++++++++------ docs/_sources/index.rst.txt | 3 +- docs/first_steps.html | 75 ++++++++++++++++------ docs/index.html | 103 ++++++++++++++++++------------ docs/searchindex.js | 2 +- sphinx/source/first_steps.rst | 79 +++++++++++++++++------ sphinx/source/index.rst | 3 +- 8 files changed, 248 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e93ee..2a5876e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## [Unreleased] -- metric metadata accessors fall back to the properties a metric inherits (under "inherited" in the metric metadata of the current convention), can be deactivated via keyword argument include_inherited=False +- metadata accessors can fall back to the properties a metric inherits from objects higher up in an asset hierarchy (under "inherited" in the metric metadata of the current convention), only as last resort after "value_dimensions_shared". This is the default for get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info. The functions for dedicated information (units, names, display names, short display names and measurements) only do so with keyword argument include_inherited=True, since an inherited property with the same label usually does not describe the metric itself +- get_metric_info and get_singlets_metric_info fall back to the value dimension "value" and to "value_dimensions_shared" if the metric itself does not provide the information +- fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension +- fix: get_metric_info returns None entries instead of raising if the metadata contains no metric information +- the series accessors use the only metric if "single_metric" is missing in the dataset metadata ## [0.0.9] - 2026-09-10 - singletsframe support diff --git a/docs/_sources/first_steps.rst.txt b/docs/_sources/first_steps.rst.txt index c2a396f..1f90038 100644 --- a/docs/_sources/first_steps.rst.txt +++ b/docs/_sources/first_steps.rst.txt @@ -5,50 +5,91 @@ First steps How to get metadata with hdhelpers? =================================== -Let's say we want to retrieve the metadata of a timeseries. +Let's say we want to retrieve the metadata of a timeseries, which follows the +`hetida designer metadata conventions`_. In hetida designer this series can be represented as json for *direct provisioning* .. code-block:: json { - "__hd_wrapped_data_object__":"SERIES", + "__hd_wrapped_data_object__": "SERIES", "__metadata__": { - "single_metric_dataset_metadata": { - "ref_interval_end_timestamp":"2020-01-01T08:20:00.000Z", - "ref_interval_start_timestamp": "2020-01-01T08:10:00.000Z" + "dataset_metadata": { + "metric_key": "external_id", + "single_metric": "water.north.level", + "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00", + "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00", + "ref_interval_type": "closed" }, - "single_metric_metadata": { - "structured_metadata": { - "metric": { - "short_display_name": "Water Level", - "unit": "cm" - } + "metrics": [ + { + "external_id": "water.north.level", + "name": "Water level north", + "short_display_name": "Water Level", + "value_dimensions": [{"column": "value", "unit": "cm"}], + "inherited": {"latitude": 51.43, "longitude": 7.58} } - } + ] }, "__data__": { "2020-01-01T08:10:00+00:00": 1, "2020-01-01T08:15:00+00:00": 2, "2020-01-01T08:16:00+00:00": 3, - "2020-01-01T08:17:00+00:00": 4, + "2020-01-01T08:17:00+00:00": 4 } } -We can retrieve the name and unit of the series with the following code +We can retrieve the name, unit and latitude of the series with the following code .. code-block:: python - from hdhelpers.metadata import get_series_name, get_series_unit + from hdhelpers.metadata import get_series_info, get_series_name, get_series_unit def main(*, series): # entrypoint function for this component # ***** DO NOT EDIT LINES ABOVE ***** - name = get_series_name(series) - unit = get_series_unit(series) + name = get_series_name(series) # "Water level north" + unit = get_series_unit(series) # "cm" + latitude = get_series_info(series, "latitude") # 51.43, inherited from the hierarchy ... +.. _hetida designer metadata conventions: https://hetida.github.io/hetida-designer/user_guide/attached_metadata/ + +Where does the information come from? +===================================== + +The metadata conventions allow to provide information on different levels. The functions +look for it in the following order and use the first information which is not null. Information +provided as null counts as missing. + +**Dedicated information** like the unit, name, display name, short display name and measurement +(e.g. ``get_units``, ``get_series_unit``, ``get_singlets_units``) is looked up per value dimension: + +1. in the value dimension itself, +2. only for the value dimension ``"value"``: in the metric, +3. in ``"value_dimensions_shared"``. + +For (short) display names, each field is looked up on all of these levels before falling back to +the next field: short display name, then display name, then name. So e.g. a display name provided +for the metric wins over the name of its value dimension. + +**Other information** is looked up by ``get_metric_info`` and ``get_singlets_metric_info`` + +1. in the metric itself, +2. in its value dimension ``"value"``, +3. in ``"value_dimensions_shared"``, +4. in the properties the metric inherits from objects higher up in an asset hierarchy (given under + ``"inherited"`` in the metric metadata), e.g. the latitude of the plant a sensor belongs to. + +``get_series_info`` and ``get_singlets_info`` use the order of the dedicated information, followed +by the inherited properties as last resort. + +The inherited properties can be switched off via the keyword argument ``include_inherited=False``. +In contrast, the functions for dedicated information do not use them by default, since an +inherited property with the same label usually does not describe the metric itself, e.g. a +property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. How to use hdhelpers for plotting? (tbd) @@ -78,10 +119,10 @@ Our component code might look like this to plot the timeseries accordingly: fig = go.Figure([go.Scatter(x=series.index, y=series.values, marker={"color": resolve_color("ki.vision")})]) - start, end = get_queried_interval(series=series) + start, end = get_queried_interval(series) fig.update_xaxes(range=(start, end)) - full_title = get_series_name(series=series, default_title="Level") + full_title = get_series_name(series) or "Level" fig.update_layout(yaxis_title=full_title) set_dt_ticks(fig) diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt index b6ca2c0..cf00ef9 100644 --- a/docs/_sources/index.rst.txt +++ b/docs/_sources/index.rst.txt @@ -14,12 +14,13 @@ hdhelpers is a package designed for and included in the standard installation of It contains functions that streamline plotting components, especially those that are used in the `hetida platform`_, by -* accessing series metadata that complies with the hetida platform metadata scheme, +* accessing metadata of series, singletsframes and multitsframes that complies with the `hetida designer metadata conventions`_, * aditional helper functions like adjusting the timezone of timestamps, series, and dataframes, * accessing metadata that the hetida platform writes into the hetida designer's *plot_target_settings* context variable (tbd), * providing toggleable standardized styling options and json serialization for plotly plots (tbd). .. _hetida designer: https://github.com/hetida/hetida-designer +.. _hetida designer metadata conventions: https://hetida.github.io/hetida-designer/user_guide/attached_metadata/ .. _hetida platform: https://hetida.io/ Further Information diff --git a/docs/first_steps.html b/docs/first_steps.html index 5796262..4c7c9d2 100644 --- a/docs/first_steps.html +++ b/docs/first_steps.html @@ -44,47 +44,83 @@

Navigation

First steps

How to get metadata with hdhelpers?

-

Let’s say we want to retrieve the metadata of a timeseries. +

Let’s say we want to retrieve the metadata of a timeseries, which follows the +hetida designer metadata conventions. In hetida designer this series can be represented as json for direct provisioning

{
-    "__hd_wrapped_data_object__":"SERIES",
+    "__hd_wrapped_data_object__": "SERIES",
     "__metadata__": {
-        "single_metric_dataset_metadata": {
-            "ref_interval_end_timestamp":"2020-01-01T08:20:00.000Z",
-            "ref_interval_start_timestamp": "2020-01-01T08:10:00.000Z"
+        "dataset_metadata": {
+            "metric_key": "external_id",
+            "single_metric": "water.north.level",
+            "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00",
+            "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00",
+            "ref_interval_type": "closed"
         },
-        "single_metric_metadata": {
-            "structured_metadata": {
-                "metric": {
-                    "short_display_name": "Water Level",
-                    "unit": "cm"
-                }
+        "metrics": [
+            {
+                "external_id": "water.north.level",
+                "name": "Water level north",
+                "short_display_name": "Water Level",
+                "value_dimensions": [{"column": "value", "unit": "cm"}],
+                "inherited": {"latitude": 51.43, "longitude": 7.58}
             }
-        }
+        ]
     },
     "__data__": {
         "2020-01-01T08:10:00+00:00": 1,
         "2020-01-01T08:15:00+00:00": 2,
         "2020-01-01T08:16:00+00:00": 3,
-        "2020-01-01T08:17:00+00:00": 4,
+        "2020-01-01T08:17:00+00:00": 4
     }
 }
-

We can retrieve the name and unit of the series with the following code

-
from hdhelpers.metadata import get_series_name, get_series_unit
+
We can retrieve the name, unit and latitude of the series with the following code

+
from hdhelpers.metadata import get_series_info, get_series_name, get_series_unit
 
 def main(*, series):
     # entrypoint function for this component
     # ***** DO NOT EDIT LINES ABOVE *****
 
-    name = get_series_name(series)
-    unit = get_series_unit(series)
+    name = get_series_name(series)  # "Water level north"
+    unit = get_series_unit(series)  # "cm"
+    latitude = get_series_info(series, "latitude")  # 51.43, inherited from the hierarchy
 
     ...
+
+

Where does the information come from?

+

The metadata conventions allow to provide information on different levels. The functions +look for it in the following order and use the first information which is not null. Information +provided as null counts as missing.

+

Dedicated information like the unit, name, display name, short display name and measurement +(e.g. get_units, get_series_unit, get_singlets_units) is looked up per value dimension:

+
    +

  1. in the value dimension itself,

    2. +

  2. only for the value dimension "value": in the metric,

    3. +

  3. in "value_dimensions_shared".

    4. +
+

For (short) display names, each field is looked up on all of these levels before falling back to +the next field: short display name, then display name, then name. So e.g. a display name provided +for the metric wins over the name of its value dimension.

+

Other information is looked up by get_metric_info and get_singlets_metric_info

+
    +

  1. in the metric itself,

    2. +

  2. in its value dimension "value",

    3. +

  3. in "value_dimensions_shared",

    4. +

  4. in the properties the metric inherits from objects higher up in an asset hierarchy (given under +"inherited" in the metric metadata), e.g. the latitude of the plant a sensor belongs to.

    5. +
+

get_series_info and get_singlets_info use the order of the dedicated information, followed +by the inherited properties as last resort.

+

The inherited properties can be switched off via the keyword argument include_inherited=False. +In contrast, the functions for dedicated information do not use them by default, since an +inherited property with the same label usually does not describe the metric itself, e.g. a +property “unit” of a plant. Pass include_inherited=True to use them as last resort anyway.

+

How to use hdhelpers for plotting? (tbd)

Let’s say we want to plot the same timeseries above using hdhelpers functionalities. @@ -107,10 +143,10 @@

How to use hdhelpers for plotting? (tbd)fig = go.Figure([go.Scatter(x=series.index, y=series.values, marker={"color": resolve_color("ki.vision")})]) - start, end = get_queried_interval(series=series) + start, end = get_queried_interval(series) fig.update_xaxes(range=(start, end)) - full_title = get_series_name(series=series, default_title="Level") + full_title = get_series_name(series) or "Level" fig.update_layout(yaxis_title=full_title) set_dt_ticks(fig) @@ -134,6 +170,7 @@

Table of Contents

  • First steps
    • diff --git a/docs/index.html b/docs/index.html index 2513aee..711d539 100644 --- a/docs/index.html +++ b/docs/index.html @@ -47,7 +47,7 @@

    Introductionhetida designer.

    It contains functions that streamline plotting components, especially those that are used in the hetida platform, by

      -

    • accessing series metadata that complies with the hetida platform metadata scheme,

      • +

    • accessing metadata of series, singletsframes and multitsframes that complies with the hetida designer metadata conventions,

    • aditional helper functions like adjusting the timezone of timestamps, series, and dataframes,

    • accessing metadata that the hetida platform writes into the hetida designer’s plot_target_settings context variable (tbd),

    • providing toggleable standardized styling options and json serialization for plotly plots (tbd).

      • @@ -59,6 +59,7 @@

      Further InformationFirst steps @@ -74,18 +75,21 @@

      Functions
      -hdhelpers.metadata.get_display_names(multitsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_display_names(multitsframe, *, include_inherited=False)

      Gets display names of the MTSF metrics from the metadata

      Parameters:

      • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:

      Dictionary of metrics containing the display names. -If the display name of the metrics is not present it returns the result of hdhelpers.metadata.get_names().

      +If the display name of the metrics is not present it returns the result of hdhelpers.metadata.get_names(). +For metadata following the current convention, the display name is looked up on the value dimension, on +the metric (for the value dimension “value”) and in “value_dimensions_shared” before falling back to the +name. Fields provided as null count as missing.

      Return type:

      defaultdict[str, defaultdict[str, str | None]]

      @@ -122,13 +126,13 @@

      Functions
      -hdhelpers.metadata.get_measurements(multitsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_measurements(multitsframe, *, include_inherited=False)

      Gets measurement (type) of the MTSF metrics from the metadata

      Parameters:

      • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -159,16 +163,23 @@

      Functionsget_units() or +get_display_names() do not fall back to inherited properties by default.

      Parameters:

      • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

      • metric_info (str | Spec) – Name of information to retrieve. Note that metric_info is interpreted as a glom Spec.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the metric, its value dimension “value” nor “value_dimensions_shared” provide the information. Defaults to True.

      Returns:
      -

      Dictionary, where keys are the entries of the metrics metadata specified via “metric_key” in “dataset_metadata” and values are the entries specified via “metric_info” in the metrics metadata

      +

      Dictionary, where keys are the entries of the metrics metadata specified via “metric_key” in “dataset_metadata” and values are the entries specified via “metric_info” in the metrics metadata. Defaults to None, also if there is no metric metadata at all.

      Return type:

      defaultdict[str, Any]

      @@ -219,13 +230,13 @@

      Functions
      -hdhelpers.metadata.get_names(multitsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_names(multitsframe, *, include_inherited=False)

      Gets names of the MTSF metrics from the metadata

      Parameters:

      • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -304,13 +315,13 @@

      Functions
      -hdhelpers.metadata.get_series_display_name(series, *, include_inherited=True)
      +hdhelpers.metadata.get_series_display_name(series, *, include_inherited=False)

      Gets display name of the series from metadata

      Parameters:

      • series (pd.Series) – Series with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -345,7 +356,7 @@

      Functions

      • series (pd.Series) – Series with metadata following the convention.

      • value_dim_info (str | Spec) – Name of information to retrieve. Note that value_dim_info is interpreted as a glom Spec.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to True.

      Returns:
      @@ -368,13 +379,13 @@

      Functions
      -hdhelpers.metadata.get_series_measurement(series, *, include_inherited=True)
      +hdhelpers.metadata.get_series_measurement(series, *, include_inherited=False)

      Gets measurement (type) of the Series from metadata

      Parameters:

      • series (pd.Series) – Series with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -399,13 +410,13 @@

      Functions
      -hdhelpers.metadata.get_series_name(series, *, include_inherited=True)
      +hdhelpers.metadata.get_series_name(series, *, include_inherited=False)

      Gets name of the series from metadata

      Parameters:

      • series (pd.Series) – Series with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -446,13 +457,13 @@

      Functions
      -hdhelpers.metadata.get_series_short_display_name(series, *, include_inherited=True)
      +hdhelpers.metadata.get_series_short_display_name(series, *, include_inherited=False)

      Gets short display name of the Series from metadata

      Parameters:

      • series (pd.Series) – Series with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -478,13 +489,13 @@

      Functions
      -hdhelpers.metadata.get_series_unit(series, *, include_inherited=True)
      +hdhelpers.metadata.get_series_unit(series, *, include_inherited=False)

      Gets name of the series from metadata

      Parameters:

      • series (pd.Series) – Series with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -522,18 +533,21 @@

      Functions
      -hdhelpers.metadata.get_short_display_names(multitsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_short_display_names(multitsframe, *, include_inherited=False)

      Gets short display names of the MTSF metrics from the metadata

      Parameters:

      • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:

      Dictionary of metrics containing the short display names. -If the short display name of the metrics is not present it returns the result of hdhelpers.metadata.get_display_names().

      +If the short display name of the metrics is not present it returns the result of hdhelpers.metadata.get_display_names(). +For metadata following the current convention, each of short display name, display name and name is +looked up on the value dimension, on the metric (for the value dimension “value”) and in +“value_dimensions_shared” before falling back to the next one. Fields provided as null count as missing.

      Return type:

      defaultdict[str, defaultdict[str, str | None]]

      @@ -573,14 +587,17 @@

      Functions
      -hdhelpers.metadata.get_singlets_display_names(singletsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_singlets_display_names(singletsframe, *, include_inherited=False)

      Gets display names of the value dimensions of a SingleTSFrame from metadata

      -

      Falls back to the name of the value dimension if no display name is given.

      +

      Falls back to the name of the value dimension if no display name is given. The display name +is looked up on the value dimension, on the metric (for the value dimension “value”) and in +“value_dimensions_shared” before falling back to the name. Fields provided as null count as +missing.

      Parameters:

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -630,7 +647,7 @@

      Functions

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

      • value_dim_info (str | Spec) – Name of information to retrieve. Note that value_dim_info is interpreted as a glom Spec.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to True.

      Returns:
      @@ -667,13 +684,13 @@

      Functions
      -hdhelpers.metadata.get_singlets_measurements(singletsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_singlets_measurements(singletsframe, *, include_inherited=False)

      Gets measurements (types) of the value dimensions of a SingleTSFrame from metadata

      Parameters:

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -709,13 +726,15 @@

      Functionshdhelpers.metadata.get_singlets_metric_info(singletsframe, metric_info, *, include_inherited=True)

      Gets metadata associated to the single metric of a SingleTSFrame

      Counterpart of get_metric_info() for SingleTSFrames: instead of a dictionary keyed -by metric this directly returns the requested information for the one metric.

      +by metric this directly returns the requested information for the one metric. Like +get_metric_info(), it falls back to the value dimension “value”, to +“value_dimensions_shared” and finally to the properties the metric inherits.

      Parameters:

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

      • metric_info (str | Spec) – Name of information to retrieve. Note that metric_info is interpreted as a glom Spec.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the metric, its value dimension “value” nor “value_dimensions_shared” provide the information. Defaults to True.

      Returns:
      @@ -749,13 +768,13 @@

      Functions
      -hdhelpers.metadata.get_singlets_names(singletsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_singlets_names(singletsframe, *, include_inherited=False)

      Gets names of the value dimensions of a SingleTSFrame from metadata

      Parameters:

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -792,15 +811,17 @@

      Functions
      -hdhelpers.metadata.get_singlets_short_display_names(singletsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_singlets_short_display_names(singletsframe, *, include_inherited=False)

      Gets short display names of the value dimensions of a SingleTSFrame from metadata

      Falls back to the display name and then to the name of the value dimension if no short -display name is given.

      +display name is given. Each of them is looked up on the value dimension, on the metric (for +the value dimension “value”) and in “value_dimensions_shared” before falling back to the next +one. Fields provided as null count as missing.

      Parameters:

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -837,13 +858,13 @@

      Functions
      -hdhelpers.metadata.get_singlets_units(singletsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_singlets_units(singletsframe, *, include_inherited=False)

      Gets units of the value dimensions of a SingleTSFrame from metadata

      Parameters:

      • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      @@ -895,13 +916,13 @@

      Functions
      -hdhelpers.metadata.get_units(multitsframe, *, include_inherited=True)
      +hdhelpers.metadata.get_units(multitsframe, *, include_inherited=False)

      Gets unit of value dimensions in MTSF metrics from Metadata

      Parameters:

      • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

        • -

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata), if the metric itself does not provide the information. Defaults to True.

        • +

      • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to False, since this information should be provided for the metric itself.

      Returns:
      diff --git a/docs/searchindex.js b/docs/searchindex.js index cdd0c31..7c3f85d 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"000z":0,"00z":1,"01t01":1,"01t08":0,"01t09":[],"01t10":[],"05t13":1,"06t13":1,"5s":[],"80b0ec":[],"A":1,"All":[],"DO":0,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Our":0,"That":[],"The":1,"This":1,"To":1,"We":0,"With":[],"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"activ":[],"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":1,"also":[],"ambigu":1,"ani":1,"anoth":1,"appli":[],"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"asset":1,"associ":1,"assum":1,"attr":1,"avail":1,"awar":[],"axe":1,"axi":0,"back":1,"background":1,"background_color":[],"base":1,"becaus":[],"behaviour":1,"berlin":1,"bool":1,"boolean":1,"brand":[],"by_metr":1,"c":1,"can":[0,1],"carri":1,"case":1,"certain":1,"chosen":[],"class":1,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":1,"column_nam":1,"communic":[],"compli":1,"compon":[0,1],"concret":1,"configur":[],"contain":1,"context":1,"contrast":1,"control":1,"convent":1,"convert":1,"convert_index":1,"convey":[],"correspond":[0,1],"counterpart":1,"current":[],"custom":1,"cut":[],"danger":[],"dark":1,"data":1,"datafram":1,"dataset_metadata":1,"date":1,"datetim":1,"de":1,"de_d":1,"def":0,"default":1,"default_titl":0,"default_unit":[],"defaultdict":1,"defin":[0,1],"design":[0,1],"detail":[],"determin":1,"dict":1,"dictionari":1,"dimens":1,"direct":[0,1],"display":1,"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"draw":[],"e":1,"ea1255":1,"eas":1,"eb6962":1,"edg":[],"edit":0,"els":[],"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":1,"error_cod":1,"error_color":[],"especi":1,"europ":1,"exact":1,"exampl":[0,1],"exclus":1,"execcontext":[],"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":1,"external_second":1,"extra_inform":1,"extract":1,"fail":[],"fall":1,"featur":[],"fig":[0,1],"figur":[0,1],"first":1,"flag":[],"follow":[0,1],"format":1,"four":[],"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":1,"generat":1,"german":1,"get":1,"get_and_pad_start_and_end_timestamp":[],"get_colors_from_plot_target_set":[],"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":1,"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":1,"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":1,"get_singlets_measur":1,"get_singlets_metric_info":1,"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":1,"get_unit":1,"get_value_dimension_info":1,"get_y_axis_label":[],"given":1,"global":1,"glom":1,"go":[0,1],"goal":[],"good":1,"graph_object":[0,1],"half":[],"handl":1,"happen":1,"hardwir":[],"helper":0,"helperexcept":1,"hetida":[0,1],"hide":[],"hierarchi":1,"higher":1,"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":1,"index":[0,1],"infer":1,"info":1,"info_color":[],"inform":[],"inherit":1,"inpir":[],"input":[],"insoformat":[],"inspir":1,"instal":1,"instead":1,"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"intend":[],"interpret":1,"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"keep":[],"key":1,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"load":[],"local":1,"look":0,"m":1,"magnitud":[],"main":0,"make":[],"mani":1,"marker":0,"meaning":1,"measur":1,"mention":[],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":1,"might":0,"miss":1,"mode":[],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":1,"naiv":[],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"non":1,"none":1,"note":1,"object":1,"object_to_convert":1,"one":1,"onli":1,"oper":1,"option":1,"orang":1,"order":[],"otherwis":[],"output":1,"packag":1,"panda":1,"paramet":1,"pars":[],"pass":[],"pd":1,"place":1,"platform":1,"pleas":1,"plot":[],"plot_target_set":1,"plot_target_styl":[],"plot_target_timezon":[],"plotly_fig_to_json_dict":[],"point":[],"possibl":1,"potenti":[],"precis":[],"present":1,"print":1,"properti":1,"provid":1,"provis":0,"purpos":[],"put":[],"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"reason":[],"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"reflect":[],"repres":0,"request":1,"resolv":1,"resolve_color":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"runtim":[],"s":[0,1],"say":0,"scatter":[0,1],"scheme":1,"second":1,"see":1,"sensibl":1,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_language_agnostic_datetime_tick":[],"sever":[],"shade":1,"short":1,"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":1,"singl":1,"single_metr":1,"single_metric_dataset_metadata":0,"single_metric_metadata":[0,1],"singletsfram":1,"spec":1,"specif":[],"specifi":1,"standard":[0,1],"start":[0,1],"start_pad":[],"state":1,"status":[],"status_color":[],"step":1,"str":1,"streamlin":1,"string":1,"structured_metadata":[0,1],"stsf":1,"style":1,"success_color":[],"sure":[],"surpress":1,"switch":[],"t":1,"take":[],"target":1,"tbd":1,"tech":[],"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"theme_agnostic_templ":[],"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"titl":[],"to_datetim":1,"to_timezon":1,"togeth":[],"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":1,"tupl":1,"turn":[],"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"usag":[],"use":1,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":1,"value_dimensions_shar":1,"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":1,"vision":0,"want":0,"warn_color":[],"water":0,"way":1,"well":1,"whether":1,"whose":1,"will":1,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":1,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"exampl":[],"except":1,"explan":[],"first":0,"full":[],"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":1,"introduct":1,"metadata":[0,1],"plot":[0,1],"step":0,"tbd":0,"use":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":1,"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":1,"ambigu":1,"ani":1,"anoth":1,"anyway":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"can":[0,1],"carri":1,"case":1,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"compli":1,"compon":[0,1],"concret":1,"contain":1,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":1,"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":1,"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":1,"error_cod":1,"especi":1,"europ":1,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":1,"go":[0,1],"good":1,"graph_object":[0,1],"handl":1,"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":1,"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":1,"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":1,"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mani":1,"marker":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":1,"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"next":[0,1],"non":1,"none":1,"north":0,"note":1,"null":[0,1],"object":[0,1],"object_to_convert":1,"one":1,"onli":[0,1],"oper":1,"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"pd":1,"per":0,"place":1,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":1,"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"repres":0,"request":1,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"spec":1,"specifi":1,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":1,"streamlin":1,"string":1,"structured_metadata":1,"stsf":1,"style":1,"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whose":1,"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":1,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"plot":[0,1],"step":0,"tbd":0,"use":0}}) \ No newline at end of file diff --git a/sphinx/source/first_steps.rst b/sphinx/source/first_steps.rst index c2a396f..1f90038 100644 --- a/sphinx/source/first_steps.rst +++ b/sphinx/source/first_steps.rst @@ -5,50 +5,91 @@ First steps How to get metadata with hdhelpers? =================================== -Let's say we want to retrieve the metadata of a timeseries. +Let's say we want to retrieve the metadata of a timeseries, which follows the +`hetida designer metadata conventions`_. In hetida designer this series can be represented as json for *direct provisioning* .. code-block:: json { - "__hd_wrapped_data_object__":"SERIES", + "__hd_wrapped_data_object__": "SERIES", "__metadata__": { - "single_metric_dataset_metadata": { - "ref_interval_end_timestamp":"2020-01-01T08:20:00.000Z", - "ref_interval_start_timestamp": "2020-01-01T08:10:00.000Z" + "dataset_metadata": { + "metric_key": "external_id", + "single_metric": "water.north.level", + "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00", + "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00", + "ref_interval_type": "closed" }, - "single_metric_metadata": { - "structured_metadata": { - "metric": { - "short_display_name": "Water Level", - "unit": "cm" - } + "metrics": [ + { + "external_id": "water.north.level", + "name": "Water level north", + "short_display_name": "Water Level", + "value_dimensions": [{"column": "value", "unit": "cm"}], + "inherited": {"latitude": 51.43, "longitude": 7.58} } - } + ] }, "__data__": { "2020-01-01T08:10:00+00:00": 1, "2020-01-01T08:15:00+00:00": 2, "2020-01-01T08:16:00+00:00": 3, - "2020-01-01T08:17:00+00:00": 4, + "2020-01-01T08:17:00+00:00": 4 } } -We can retrieve the name and unit of the series with the following code +We can retrieve the name, unit and latitude of the series with the following code .. code-block:: python - from hdhelpers.metadata import get_series_name, get_series_unit + from hdhelpers.metadata import get_series_info, get_series_name, get_series_unit def main(*, series): # entrypoint function for this component # ***** DO NOT EDIT LINES ABOVE ***** - name = get_series_name(series) - unit = get_series_unit(series) + name = get_series_name(series) # "Water level north" + unit = get_series_unit(series) # "cm" + latitude = get_series_info(series, "latitude") # 51.43, inherited from the hierarchy ... +.. _hetida designer metadata conventions: https://hetida.github.io/hetida-designer/user_guide/attached_metadata/ + +Where does the information come from? +===================================== + +The metadata conventions allow to provide information on different levels. The functions +look for it in the following order and use the first information which is not null. Information +provided as null counts as missing. + +**Dedicated information** like the unit, name, display name, short display name and measurement +(e.g. ``get_units``, ``get_series_unit``, ``get_singlets_units``) is looked up per value dimension: + +1. in the value dimension itself, +2. only for the value dimension ``"value"``: in the metric, +3. in ``"value_dimensions_shared"``. + +For (short) display names, each field is looked up on all of these levels before falling back to +the next field: short display name, then display name, then name. So e.g. a display name provided +for the metric wins over the name of its value dimension. + +**Other information** is looked up by ``get_metric_info`` and ``get_singlets_metric_info`` + +1. in the metric itself, +2. in its value dimension ``"value"``, +3. in ``"value_dimensions_shared"``, +4. in the properties the metric inherits from objects higher up in an asset hierarchy (given under + ``"inherited"`` in the metric metadata), e.g. the latitude of the plant a sensor belongs to. + +``get_series_info`` and ``get_singlets_info`` use the order of the dedicated information, followed +by the inherited properties as last resort. + +The inherited properties can be switched off via the keyword argument ``include_inherited=False``. +In contrast, the functions for dedicated information do not use them by default, since an +inherited property with the same label usually does not describe the metric itself, e.g. a +property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. How to use hdhelpers for plotting? (tbd) @@ -78,10 +119,10 @@ Our component code might look like this to plot the timeseries accordingly: fig = go.Figure([go.Scatter(x=series.index, y=series.values, marker={"color": resolve_color("ki.vision")})]) - start, end = get_queried_interval(series=series) + start, end = get_queried_interval(series) fig.update_xaxes(range=(start, end)) - full_title = get_series_name(series=series, default_title="Level") + full_title = get_series_name(series) or "Level" fig.update_layout(yaxis_title=full_title) set_dt_ticks(fig) diff --git a/sphinx/source/index.rst b/sphinx/source/index.rst index b6ca2c0..cf00ef9 100644 --- a/sphinx/source/index.rst +++ b/sphinx/source/index.rst @@ -14,12 +14,13 @@ hdhelpers is a package designed for and included in the standard installation of It contains functions that streamline plotting components, especially those that are used in the `hetida platform`_, by -* accessing series metadata that complies with the hetida platform metadata scheme, +* accessing metadata of series, singletsframes and multitsframes that complies with the `hetida designer metadata conventions`_, * aditional helper functions like adjusting the timezone of timestamps, series, and dataframes, * accessing metadata that the hetida platform writes into the hetida designer's *plot_target_settings* context variable (tbd), * providing toggleable standardized styling options and json serialization for plotly plots (tbd). .. _hetida designer: https://github.com/hetida/hetida-designer +.. _hetida designer metadata conventions: https://hetida.github.io/hetida-designer/user_guide/attached_metadata/ .. _hetida platform: https://hetida.io/ Further Information From 47f0945f4d7f086d86aa2d99cbe80858a5081eb5 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Wed, 21 Oct 2026 15:05:58 +0200 Subject: [PATCH 05/15] better document glom spec to sub fields in documentation --- CHANGELOG.md | 1 + docs/_sources/first_steps.rst.txt | 62 +++++++++++++++++++++++++++++++ docs/first_steps.html | 55 +++++++++++++++++++++++++++ docs/index.html | 18 +++++++-- docs/searchindex.js | 2 +- sphinx/source/first_steps.rst | 62 +++++++++++++++++++++++++++++++ src/hdhelpers/metadata/helpers.py | 19 ++++++++-- 7 files changed, 210 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a5876e..d5103fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension - fix: get_metric_info returns None entries instead of raising if the metadata contains no metric information - the series accessors use the only metric if "single_metric" is missing in the dataset metadata +- documentation: the information requested from get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info is interpreted as a glom spec. New section "How to query nested information?" with examples for dotted paths into sub dictionaries, other glom specs and keys containing dots ## [0.0.9] - 2026-09-10 - singletsframe support diff --git a/docs/_sources/first_steps.rst.txt b/docs/_sources/first_steps.rst.txt index 1f90038..4cccdfb 100644 --- a/docs/_sources/first_steps.rst.txt +++ b/docs/_sources/first_steps.rst.txt @@ -91,6 +91,68 @@ In contrast, the functions for dedicated information do not use them by default, inherited property with the same label usually does not describe the metric itself, e.g. a property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. +How to query nested information? +================================ + +The information requested from ``get_metric_info``, ``get_singlets_metric_info``, +``get_series_info`` and ``get_singlets_info`` is interpreted as a `glom`_ spec. A string is a +path in which dots separate the keys, so it also reaches into sub dictionaries. Let's say the +metric of the series above additionally provides comments and information on its sensor (the +conventions allow additional fields at every level), and the label of one of its inherited +properties contains a dot: + +.. code-block:: json + + { + "external_id": "water.north.level", + "name": "Water level north", + "sensor": { + "manufacturer": "Acme", + "calibration": {"date": "2026-03-01", "offset": 0.02} + }, + "comments": [ + {"content": "Gauge maintenance"}, + {"content": "Sensor replaced"} + ], + "value_dimensions": [{"column": "value", "unit": "cm"}], + "inherited": {"latitude": 51.43, "operator.name": "Ruhrverband"} + } + +We can retrieve the nested information with the following code + +.. code-block:: python + + from glom import Path + from hdhelpers.metadata import get_series_info + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + manufacturer = get_series_info(series, "sensor.manufacturer") # "Acme" + calibration_date = get_series_info(series, "sensor.calibration.date") # "2026-03-01" + latitude = get_series_info(series, "inherited.latitude") # 51.43, never the metric's own + firmware = get_series_info(series, "sensor.firmware") # None, not provided + + # any other glom spec, e.g. the content of each comment + comments = get_series_info(series, ("comments", ["content"])) + # ["Gauge maintenance", "Sensor replaced"] + + # a key which itself contains dots requires a glom Path + operator = get_series_info(series, Path("operator.name")) # "Ruhrverband" + + ... + +For a MULTITSFRAME, ``get_metric_info(multitsframe, "sensor.manufacturer")`` provides the +manufacturer keyed by metric accordingly. + +The fallbacks described above apply to the whole path, e.g. ``"sensor.manufacturer"`` is looked +up in the metric first, then in its value dimension ``"value"`` and so on. A path which is not +present yields None instead of an error. Note that a plain string like ``"operator.name"`` is +always split at its dots, so it does not find the inherited property ``"operator.name"``. + +.. _glom: https://glom.readthedocs.io/ + How to use hdhelpers for plotting? (tbd) ======================================== diff --git a/docs/first_steps.html b/docs/first_steps.html index 4c7c9d2..9d4beaa 100644 --- a/docs/first_steps.html +++ b/docs/first_steps.html @@ -121,6 +121,60 @@

      Where does the information come from?include_inherited=True to use them as last resort anyway.

+
+

How to query nested information?

+

The information requested from get_metric_info, get_singlets_metric_info, +get_series_info and get_singlets_info is interpreted as a glom spec. A string is a +path in which dots separate the keys, so it also reaches into sub dictionaries. Let’s say the +metric of the series above additionally provides comments and information on its sensor (the +conventions allow additional fields at every level), and the label of one of its inherited +properties contains a dot:

+
{
+    "external_id": "water.north.level",
+    "name": "Water level north",
+    "sensor": {
+        "manufacturer": "Acme",
+        "calibration": {"date": "2026-03-01", "offset": 0.02}
+    },
+    "comments": [
+        {"content": "Gauge maintenance"},
+        {"content": "Sensor replaced"}
+    ],
+    "value_dimensions": [{"column": "value", "unit": "cm"}],
+    "inherited": {"latitude": 51.43, "operator.name": "Ruhrverband"}
+}
+
+
+

We can retrieve the nested information with the following code

+
from glom import Path
+from hdhelpers.metadata import get_series_info
+
+def main(*, series):
+    # entrypoint function for this component
+    # ***** DO NOT EDIT LINES ABOVE *****
+
+    manufacturer = get_series_info(series, "sensor.manufacturer")  # "Acme"
+    calibration_date = get_series_info(series, "sensor.calibration.date")  # "2026-03-01"
+    latitude = get_series_info(series, "inherited.latitude")  # 51.43, never the metric's own
+    firmware = get_series_info(series, "sensor.firmware")  # None, not provided
+
+    # any other glom spec, e.g. the content of each comment
+    comments = get_series_info(series, ("comments", ["content"]))
+    # ["Gauge maintenance", "Sensor replaced"]
+
+    # a key which itself contains dots requires a glom Path
+    operator = get_series_info(series, Path("operator.name"))  # "Ruhrverband"
+
+    ...
+
+
+

For a MULTITSFRAME, get_metric_info(multitsframe, "sensor.manufacturer") provides the +manufacturer keyed by metric accordingly.

+

The fallbacks described above apply to the whole path, e.g. "sensor.manufacturer" is looked +up in the metric first, then in its value dimension "value" and so on. A path which is not +present yields None instead of an error. Note that a plain string like "operator.name" is +always split at its dots, so it does not find the inherited property "operator.name".

+

How to use hdhelpers for plotting? (tbd)

Let’s say we want to plot the same timeseries above using hdhelpers functionalities. @@ -171,6 +225,7 @@

Table of Contents

  • First steps
    • diff --git a/docs/index.html b/docs/index.html index 711d539..00d6836 100644 --- a/docs/index.html +++ b/docs/index.html @@ -60,6 +60,7 @@

    Further InformationFirst steps @@ -174,7 +175,7 @@

    FunctionsParameters:

    • multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

      • -

    • metric_info (str | Spec) – Name of information to retrieve. Note that metric_info is interpreted as a glom Spec.

      • +

    • metric_info (str | Spec) – Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like “sensor.manufacturer” into sub dictionaries or any other glom Spec. See “How to query nested information?” in the first steps.

    • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the metric, its value dimension “value” nor “value_dimensions_shared” provide the information. Defaults to True.

    @@ -196,6 +197,7 @@

    Functions... "external_id": "external_first", ... "unit": "m", ... "display_name": "first display name", +... "sensor": {"manufacturer": "Acme"}, ... "value_dimensions": [ ... { ... "column": "temp", @@ -226,6 +228,14 @@

    FunctionsTrue +

    A dotted path reaches into sub dictionaries:

    +
    >>> result = get_metric_info(multitsframe, "sensor.manufacturer")
+>>> result["first"]
+'Acme'
+>>> result["second"] is None
+True
+
    +

    @@ -355,7 +365,7 @@

    FunctionsParameters:

    • series (pd.Series) – Series with metadata following the convention.

      • -

    • value_dim_info (str | Spec) – Name of information to retrieve. Note that value_dim_info is interpreted as a glom Spec.

      • +

    • value_dim_info (str | Spec) – Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like “sensor.manufacturer” into sub dictionaries or any other glom Spec. See “How to query nested information?” in the first steps.

    • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to True.

    @@ -646,7 +656,7 @@

    FunctionsParameters:

    • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

      • -

    • value_dim_info (str | Spec) – Name of information to retrieve. Note that value_dim_info is interpreted as a glom Spec.

      • +

    • value_dim_info (str | Spec) – Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like “sensor.manufacturer” into sub dictionaries or any other glom Spec. See “How to query nested information?” in the first steps.

    • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor “value_dimensions_shared” provide the information. Defaults to True.

    @@ -733,7 +743,7 @@

    FunctionsParameters:

    • singletsframe (pd.DataFrame) – STSF with metadata following the convention.

      • -

    • metric_info (str | Spec) – Name of information to retrieve. Note that metric_info is interpreted as a glom Spec.

      • +

    • metric_info (str | Spec) – Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like “sensor.manufacturer” into sub dictionaries or any other glom Spec. See “How to query nested information?” in the first steps.

    • include_inherited (bool) – Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under “inherited” in the metric metadata) as last resort, i.e. if neither the metric, its value dimension “value” nor “value_dimensions_shared” provide the information. Defaults to True.

    diff --git a/docs/searchindex.js b/docs/searchindex.js index 7c3f85d..7efef15 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":1,"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":1,"ambigu":1,"ani":1,"anoth":1,"anyway":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"can":[0,1],"carri":1,"case":1,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"compli":1,"compon":[0,1],"concret":1,"contain":1,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":1,"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":1,"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":1,"error_cod":1,"especi":1,"europ":1,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":1,"go":[0,1],"good":1,"graph_object":[0,1],"handl":1,"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":1,"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":1,"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":1,"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mani":1,"marker":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":1,"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"next":[0,1],"non":1,"none":1,"north":0,"note":1,"null":[0,1],"object":[0,1],"object_to_convert":1,"one":1,"onli":[0,1],"oper":1,"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"pd":1,"per":0,"place":1,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":1,"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"repres":0,"request":1,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"spec":1,"specifi":1,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":1,"streamlin":1,"string":1,"structured_metadata":1,"stsf":1,"style":1,"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whose":1,"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":1,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"plot":[0,1],"step":0,"tbd":0,"use":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":[0,1],"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"ani":[0,1],"anoth":1,"anyway":0,"appli":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":1,"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"handl":1,"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"nest":1,"never":0,"next":[0,1],"non":1,"none":[0,1],"north":0,"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":0,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"reach":[0,1],"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"repres":0,"request":[0,1],"requir":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":1,"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whole":0,"whose":1,"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"use":0}}) \ No newline at end of file diff --git a/sphinx/source/first_steps.rst b/sphinx/source/first_steps.rst index 1f90038..4cccdfb 100644 --- a/sphinx/source/first_steps.rst +++ b/sphinx/source/first_steps.rst @@ -91,6 +91,68 @@ In contrast, the functions for dedicated information do not use them by default, inherited property with the same label usually does not describe the metric itself, e.g. a property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. +How to query nested information? +================================ + +The information requested from ``get_metric_info``, ``get_singlets_metric_info``, +``get_series_info`` and ``get_singlets_info`` is interpreted as a `glom`_ spec. A string is a +path in which dots separate the keys, so it also reaches into sub dictionaries. Let's say the +metric of the series above additionally provides comments and information on its sensor (the +conventions allow additional fields at every level), and the label of one of its inherited +properties contains a dot: + +.. code-block:: json + + { + "external_id": "water.north.level", + "name": "Water level north", + "sensor": { + "manufacturer": "Acme", + "calibration": {"date": "2026-03-01", "offset": 0.02} + }, + "comments": [ + {"content": "Gauge maintenance"}, + {"content": "Sensor replaced"} + ], + "value_dimensions": [{"column": "value", "unit": "cm"}], + "inherited": {"latitude": 51.43, "operator.name": "Ruhrverband"} + } + +We can retrieve the nested information with the following code + +.. code-block:: python + + from glom import Path + from hdhelpers.metadata import get_series_info + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + manufacturer = get_series_info(series, "sensor.manufacturer") # "Acme" + calibration_date = get_series_info(series, "sensor.calibration.date") # "2026-03-01" + latitude = get_series_info(series, "inherited.latitude") # 51.43, never the metric's own + firmware = get_series_info(series, "sensor.firmware") # None, not provided + + # any other glom spec, e.g. the content of each comment + comments = get_series_info(series, ("comments", ["content"])) + # ["Gauge maintenance", "Sensor replaced"] + + # a key which itself contains dots requires a glom Path + operator = get_series_info(series, Path("operator.name")) # "Ruhrverband" + + ... + +For a MULTITSFRAME, ``get_metric_info(multitsframe, "sensor.manufacturer")`` provides the +manufacturer keyed by metric accordingly. + +The fallbacks described above apply to the whole path, e.g. ``"sensor.manufacturer"`` is looked +up in the metric first, then in its value dimension ``"value"`` and so on. A path which is not +present yields None instead of an error. Note that a plain string like ``"operator.name"`` is +always split at its dots, so it does not find the inherited property ``"operator.name"``. + +.. _glom: https://glom.readthedocs.io/ + How to use hdhelpers for plotting? (tbd) ======================================== diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 0174f8b..106d53d 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -295,7 +295,7 @@ def get_metric_info( Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. - metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. + metric_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the metric, its value dimension "value" nor "value_dimensions_shared" provide the information. Defaults to True. Returns: @@ -315,6 +315,7 @@ def get_metric_info( ... "external_id": "external_first", ... "unit": "m", ... "display_name": "first display name", + ... "sensor": {"manufacturer": "Acme"}, ... "value_dimensions": [ ... { ... "column": "temp", @@ -343,6 +344,16 @@ def get_metric_info( 'external_second' >>> result["not-given"] is None True + + A dotted path reaches into sub dictionaries: + + .. doctest:: metadata.get_metric_info + + >>> result = get_metric_info(multitsframe, "sensor.manufacturer") + >>> result["first"] + 'Acme' + >>> result["second"] is None + True """ check_dataframe(multitsframe) return defaultdict( @@ -360,7 +371,7 @@ def get_series_info( Args: series (pd.Series): Series with metadata following the convention. - value_dim_info (str | Spec): Name of information to retrieve. Note that `value_dim_info` is interpreted as a glom Spec. + value_dim_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to True. Returns: @@ -659,7 +670,7 @@ def get_singlets_info( Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - value_dim_info (str | Spec): Name of information to retrieve. Note that `value_dim_info` is interpreted as a glom Spec. + value_dim_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to True. Returns: @@ -947,7 +958,7 @@ def get_singlets_metric_info( Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. - metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. + metric_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the metric, its value dimension "value" nor "value_dimensions_shared" provide the information. Defaults to True. Returns: From 5d67fa7f8774068bcc4b57706898488e0da3e94a Mon Sep 17 00:00:00 2001 From: stewit <> Date: Thu, 22 Oct 2026 08:52:26 +0200 Subject: [PATCH 06/15] evaluate metadata fallbacks only for metrics lacking information get_metric_info and get_singlets_metric_info applied the spec to the fallback levels of all metrics, even if a metric provided the information itself. A spec transforming the information, e.g. ("name", str.upper), failed on a null name of the value dimension although the metric had a name. Now the fallbacks are evaluated only for metrics lacking the information, and errors raised by the spec are no longer mistaken for missing metric metadata. The fallback fields of the (short) display names are likewise only looked up for metrics still lacking information, stopping as soon as all are resolved, and the inherited properties are looked up without recomputing all other levels. This reduces the extraction passes for frames with many metrics. Document that transforming specs may receive null. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/_sources/first_steps.rst.txt | 6 ++ docs/first_steps.html | 5 ++ docs/searchindex.js | 2 +- sphinx/source/first_steps.rst | 6 ++ src/hdhelpers/metadata/_helpers.py | 99 ++++++++++++++++++----- src/hdhelpers/metadata/_specs.py | 12 +++ tests/metadata/test_metadata_fallbacks.py | 52 ++++++++++++ 7 files changed, 163 insertions(+), 19 deletions(-) diff --git a/docs/_sources/first_steps.rst.txt b/docs/_sources/first_steps.rst.txt index 4cccdfb..337bada 100644 --- a/docs/_sources/first_steps.rst.txt +++ b/docs/_sources/first_steps.rst.txt @@ -151,6 +151,12 @@ up in the metric first, then in its value dimension ``"value"`` and so on. A pat present yields None instead of an error. Note that a plain string like ``"operator.name"`` is always split at its dots, so it does not find the inherited property ``"operator.name"``. +A spec may also transform the information, e.g. ``("name", str.upper)``. Since information +provided as null counts as missing, such a spec may receive null: on the metric itself and, if +the metric does not provide the information, on the levels it falls back to. Errors raised by +the spec are not caught. So either let it handle None, or rather transform the information +after retrieving it. + .. _glom: https://glom.readthedocs.io/ diff --git a/docs/first_steps.html b/docs/first_steps.html index 9d4beaa..49f42ac 100644 --- a/docs/first_steps.html +++ b/docs/first_steps.html @@ -174,6 +174,11 @@

    How to query nested information?"value" and so on. A path which is not present yields None instead of an error. Note that a plain string like "operator.name" is always split at its dots, so it does not find the inherited property "operator.name".

    +

    A spec may also transform the information, e.g. ("name", str.upper). Since information +provided as null counts as missing, such a spec may receive null: on the metric itself and, if +the metric does not provide the information, on the levels it falls back to. Errors raised by +the spec are not caught. So either let it handle None, or rather transform the information +after retrieving it.

    How to use hdhelpers for plotting? (tbd)

    diff --git a/docs/searchindex.js b/docs/searchindex.js index 7efef15..8812cb9 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":[0,1],"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"ani":[0,1],"anoth":1,"anyway":0,"appli":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":1,"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"handl":1,"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"nest":1,"never":0,"next":[0,1],"non":1,"none":[0,1],"north":0,"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":0,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":1,"rang":0,"re":1,"reach":[0,1],"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"repres":0,"request":[0,1],"requir":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":1,"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":1,"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whole":0,"whose":1,"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"use":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":[0,1],"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"ani":[0,1],"anoth":1,"anyway":0,"appli":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":1,"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"handl":[0,1],"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"may":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"nest":1,"never":0,"next":[0,1],"non":1,"none":[0,1],"north":0,"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":0,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"repres":0,"request":[0,1],"requir":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whole":0,"whose":1,"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"use":0}}) \ No newline at end of file diff --git a/sphinx/source/first_steps.rst b/sphinx/source/first_steps.rst index 4cccdfb..337bada 100644 --- a/sphinx/source/first_steps.rst +++ b/sphinx/source/first_steps.rst @@ -151,6 +151,12 @@ up in the metric first, then in its value dimension ``"value"`` and so on. A pat present yields None instead of an error. Note that a plain string like ``"operator.name"`` is always split at its dots, so it does not find the inherited property ``"operator.name"``. +A spec may also transform the information, e.g. ``("name", str.upper)``. Since information +provided as null counts as missing, such a spec may receive null: on the metric itself and, if +the metric does not provide the information, on the levels it falls back to. Errors raised by +the spec are not caught. So either let it handle None, or rather transform the information +after retrieving it. + .. _glom: https://glom.readthedocs.io/ diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index f9c528b..a40a563 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -2,7 +2,7 @@ from typing import Any import pandas as pd -from glom import Check, Coalesce, GlomError, Spec, glom +from glom import Check, Coalesce, CoalesceError, Spec, glom import hdhelpers.metadata._specs as _specs @@ -61,18 +61,52 @@ def get_value_dimension_info( For examples we refer to the corresponding unit tests (/tests/helpers/test_metadata.py). """ - spec = _specs.by_metric_key_by_val_dimension(value_dim_info, include_inherited) - value_dimension_info_by_metric_by_value_dimension = glom(multitsframe.attrs, spec) return defaultdict( - lambda: defaultdict(lambda: None), value_dimension_info_by_metric_by_value_dimension + lambda: defaultdict(lambda: None), + _value_dimension_info_of_attrs(multitsframe.attrs, value_dim_info, include_inherited), ) +def _value_dimension_info_of_attrs( + attrs: Any, value_dim_info: str | Spec, include_inherited: bool = False +) -> Any: + spec = _specs.by_metric_key_by_val_dimension(value_dim_info, include_inherited) + return glom(attrs, spec) + + def follows_current_convention(attrs: Any) -> bool: """Whether metadata follows the current metadata convention, i.e. has a list of metrics""" return isinstance(attrs, dict) and isinstance(attrs.get("metrics"), list) +def _restricted_to_metrics(attrs: dict, metrics: set) -> dict: + """The metadata with only the metric objects of the given metrics + + The information of a metric is extracted independently of the other metrics, so fallbacks + only need to be evaluated for the metrics still lacking information. Besides saving time + for frames with many metrics, this avoids applying a spec to fallback levels whose values + (e.g. null) the spec cannot handle, while the metric itself provides the information. + + Only for metadata following the current metadata convention. + """ + metric_key = extract_from_metadata(attrs, key="metric_key", default="id") + return attrs | { + "metrics": [ + metric + for metric in attrs["metrics"] + if isinstance(metric, dict) and metric.get(metric_key) in metrics + ] + } + + +def _metrics_lacking_info(info_by_metric_by_value_dimension: dict) -> set: + return { + metric + for metric, info_by_value_dimension in info_by_metric_by_value_dimension.items() + if any(info is None for info in info_by_value_dimension.values()) + } + + def value_dimension_info_with_fallbacks( data: pd.DataFrame | pd.Series, fields: tuple[str, ...], @@ -94,16 +128,37 @@ def value_dimension_info_with_fallbacks( return get_value_dimension_info(data, legacy_spec, include_inherited) result: defaultdict[str, defaultdict[str, Any]] = defaultdict(lambda: defaultdict(lambda: None)) - # inherited properties only after all fields of the metadata itself - for inherited in [False, True] if include_inherited else [False]: + attrs = data.attrs + for field in fields: + info_by_metric = _value_dimension_info_of_attrs(attrs, Coalesce(field, default=None)) + for metric, info_by_value_dimension in info_by_metric.items(): + for value_dimension, info in info_by_value_dimension.items(): + if result[metric][value_dimension] is None: + result[metric][value_dimension] = info + + # further fields only for the metrics still lacking information + metrics_lacking_info = _metrics_lacking_info(result) + if len(metrics_lacking_info) == 0: + return result + attrs = _restricted_to_metrics(data.attrs, metrics_lacking_info) + + if include_inherited: + # inherited properties only after all fields of the metadata itself and, as in + # get_value_dimension_info, only for the value dimension "value" for field in fields: - info_by_metric = get_value_dimension_info( - data, Coalesce(field, default=None), inherited + metrics_lacking_value_info = { + metric + for metric, info_by_value_dimension in result.items() + if info_by_value_dimension["value"] is None + } + if len(metrics_lacking_value_info) == 0: + break + inherited_by_metric = glom( + _restricted_to_metrics(data.attrs, metrics_lacking_value_info), + _specs.inherited_by_metric_key(Coalesce(field, default=None)), ) - for metric, info_by_value_dimension in info_by_metric.items(): - for value_dimension, info in info_by_value_dimension.items(): - if result[metric][value_dimension] is None: - result[metric][value_dimension] = info + for metric in metrics_lacking_value_info: + result[metric]["value"] = inherited_by_metric.get(metric) return result @@ -175,23 +230,31 @@ def metric_info_by_metric( For metadata following the current metadata convention, information the metric itself does not provide is taken from its value dimension "value", then from "value_dimensions_shared" and, if include_inherited, finally from the properties the metric inherits. Information - provided as null counts as missing. + provided as null counts as missing. The fallbacks are only evaluated for metrics lacking the + information, see _restricted_to_metrics. - Returns an empty dict if no metric metadata is present at all. + Returns an empty dict if no metric metadata is present at all. Errors raised by applying + metric_info, e.g. a transformation failing on null, are not caught. """ try: info_by_metric = glom(data.attrs, _specs.spec_by_metric_key(metric_info)) - except GlomError: # no metric metadata present at all + except CoalesceError: # no metric metadata present at all return {} if not follows_current_convention(data.attrs): return info_by_metric - value_dimension_info = get_value_dimension_info( - data, Coalesce(metric_info, default=None), include_inherited + metrics_lacking_info = {metric for metric, info in info_by_metric.items() if info is None} + if len(metrics_lacking_info) == 0: + return info_by_metric + + value_dimension_info = _value_dimension_info_of_attrs( + _restricted_to_metrics(data.attrs, metrics_lacking_info), + Coalesce(metric_info, default=None), + include_inherited, ) return { - metric: info if info is not None else value_dimension_info[metric].get("value") + metric: info if info is not None else value_dimension_info.get(metric, {}).get("value") for metric, info in info_by_metric.items() } diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index c684709..dec0ed1 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -113,6 +113,18 @@ def _spec_inherited_by_metric(metadatum_key: str | Spec) -> Spec: ) +def inherited_by_metric_key(metadatum_key: str | Spec) -> Spec: + """Provides glom spec that extracts a metadatum from the properties each metric inherits + + Only for the current metadata convention. Returns a dict keyed by metric, which is empty if + there is no list of metrics. + """ + return ( + {"metric_key": _spec_metric_key(), "by_metric": _spec_inherited_by_metric(metadatum_key)}, + "by_metric", + ) + + def _spec_actual_per_metric_per_value_dimensions(metadatum_key: str | Spec) -> Spec: """Deliberately without a default: this is what discriminates the metadata conventions diff --git a/tests/metadata/test_metadata_fallbacks.py b/tests/metadata/test_metadata_fallbacks.py index 05e7023..f278125 100644 --- a/tests/metadata/test_metadata_fallbacks.py +++ b/tests/metadata/test_metadata_fallbacks.py @@ -22,6 +22,7 @@ get_series_unit, get_short_display_names, get_singlets_display_names, + get_singlets_metric_info, get_singlets_short_display_names, get_singlets_units, get_units, @@ -225,3 +226,54 @@ def test_metric_info_from_platform(): assert aggregation_methods["level"] == "SUM" assert aggregation_methods["flow"] is None + + +def test_metric_info_falls_back_only_for_metrics_lacking_it(): + """A spec transforming the information is not applied to fallback levels needlessly + + The null name of the value dimension of metric "a" would let str.upper fail. + """ + metric_b = metric(id="b", value_dimensions=[value_dimension(name="value name")]) + del metric_b["name"] + multitsframe = frame({"metrics": [metric(id="a", name="Sensor"), metric_b]}) + + names = get_metric_info(multitsframe, ("name", str.upper)) + + assert names["a"] == "SENSOR" + assert names["b"] == "VALUE NAME" + assert get_singlets_metric_info(frame(platform_series_attrs), ("name", str.upper)) == ( + "WATER-LVL-NORTH" + ) + + +def test_metric_info_raises_errors_of_the_spec(): + """Errors of the spec must not be mistaken for missing metric metadata""" + with pytest.raises(TypeError): + get_metric_info(frame(platform_multitsframe_attrs), ("display_name", str.upper)) + + +@pytest.mark.parametrize("include_inherited", [False, True]) +def test_short_display_names_of_metrics_resolved_by_different_fields(include_inherited): + """Each metric gets its information from the first field providing it, independently""" + attrs = { + "metrics": [ + metric( + id="a", + short_display_name="a short", + value_dimensions=[value_dimension(), value_dimension(column="state")], + ), + metric(id="b", display_name="b display", name="b name"), + metric(id="c", name="c name"), + metric(id="d", inherited={"display_name": "d inherited"}), + ], + "value_dimensions_shared": [{"column": "state", "name": "shared state"}], + } + + short_display_names = get_short_display_names(frame(attrs), include_inherited=include_inherited) + + assert short_display_names["a"]["value"] == "a short" + assert short_display_names["a"]["state"] == "shared state" + assert short_display_names["b"]["value"] == "b display" + assert short_display_names["c"]["value"] == "c name" + assert short_display_names["c"]["state"] == "shared state" + assert short_display_names["d"]["value"] == ("d inherited" if include_inherited else None) From fc709229a546d5020927b14f0cd2cfa3b664efd4 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Thu, 22 Oct 2026 09:56:29 +0200 Subject: [PATCH 07/15] extract metadata of the current convention in a single pass Evaluating glom specs costs a lot of time per metric, which added up for frames with many metrics, since the fallbacks required an extraction per level and field. Metadata following the current convention is now gathered once in plain Python and the fallback chains are resolved on the gathered objects. Field names and dotted paths are looked up without glom, other specs are still evaluated by glom. E.g. short display names of 10000 metrics take 0.017s instead of 3.9s. The requested information is only looked up on the levels needed, so a spec transforming the information is no longer applied to null values of unused levels, e.g. of inherited properties. Entries not adhering to the conventions are skipped: metrics which are no objects or whose metric key field is no string, and value dimensions which are no objects or whose column is no string. Before, a single such entry could make all information missing or raise. The glom specs remain for the older metadata structures only. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 + docs/_sources/first_steps.rst.txt | 5 + docs/first_steps.html | 4 + docs/searchindex.js | 2 +- sphinx/source/first_steps.rst | 5 + src/hdhelpers/metadata/_current_convention.py | 210 +++++++++++++++ src/hdhelpers/metadata/_helpers.py | 117 ++------ src/hdhelpers/metadata/_specs.py | 250 +----------------- src/hdhelpers/metadata/helpers.py | 2 +- .../test_metadata_current_convention.py | 179 +++++++++++++ 10 files changed, 441 insertions(+), 336 deletions(-) create mode 100644 src/hdhelpers/metadata/_current_convention.py create mode 100644 tests/metadata/test_metadata_current_convention.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d5103fc..dc2895a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ - fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension - fix: get_metric_info returns None entries instead of raising if the metadata contains no metric information - the series accessors use the only metric if "single_metric" is missing in the dataset metadata +- metadata following the current convention is extracted in a single pass in plain Python instead of evaluating glom specs per level and field. The accessors are considerably faster for frames with many metrics, e.g. about 100 times for 1000 metrics when requesting field names or dotted paths. Other glom specs are still evaluated by glom, but only on the levels needed +- the requested information is only looked up on the levels needed, so e.g. a spec transforming the information is no longer applied to null values of levels which are not used anyway +- entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error - documentation: the information requested from get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info is interpreted as a glom spec. New section "How to query nested information?" with examples for dotted paths into sub dictionaries, other glom specs and keys containing dots ## [0.0.9] - 2026-09-10 diff --git a/docs/_sources/first_steps.rst.txt b/docs/_sources/first_steps.rst.txt index 337bada..13aded9 100644 --- a/docs/_sources/first_steps.rst.txt +++ b/docs/_sources/first_steps.rst.txt @@ -91,6 +91,11 @@ In contrast, the functions for dedicated information do not use them by default, inherited property with the same label usually does not describe the metric itself, e.g. a property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. +Entries which do not adhere to the conventions are skipped: metrics which are no objects or +whose field named by the metric key is no string, and value dimensions which are no objects or +whose column is no string. ``"value_dimensions"`` and ``"value_dimensions_shared"`` which are no +lists count as empty, a metric key which is no string as missing, i.e. ``"id"`` is used. + How to query nested information? ================================ diff --git a/docs/first_steps.html b/docs/first_steps.html index 49f42ac..93757ab 100644 --- a/docs/first_steps.html +++ b/docs/first_steps.html @@ -120,6 +120,10 @@

    Where does the information come from?include_inherited=True to use them as last resort anyway.

    +

    Entries which do not adhere to the conventions are skipped: metrics which are no objects or +whose field named by the metric key is no string, and value dimensions which are no objects or +whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no +lists count as empty, a metric key which is no string as missing, i.e. "id" is used.

    How to query nested information?

    diff --git a/docs/searchindex.js b/docs/searchindex.js index 8812cb9..5e2605a 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":[0,1],"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"ani":[0,1],"anoth":1,"anyway":0,"appli":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":1,"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":1,"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":1,"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"handl":[0,1],"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":1,"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":1,"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"may":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"nest":1,"never":0,"next":[0,1],"non":1,"none":[0,1],"north":0,"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":0,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"repres":0,"request":[0,1],"requir":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whole":0,"whose":1,"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"use":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":[0,1],"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adher":0,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"ani":[0,1],"anoth":1,"anyway":0,"appli":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":[0,1],"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":[0,1],"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":1,"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"handl":[0,1],"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":[0,1],"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":[0,1],"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"may":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"nest":1,"never":0,"next":[0,1],"non":1,"none":[0,1],"north":0,"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":0,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"repres":0,"request":[0,1],"requir":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"skip":0,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whole":0,"whose":[0,1],"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"use":0}}) \ No newline at end of file diff --git a/sphinx/source/first_steps.rst b/sphinx/source/first_steps.rst index 337bada..13aded9 100644 --- a/sphinx/source/first_steps.rst +++ b/sphinx/source/first_steps.rst @@ -91,6 +91,11 @@ In contrast, the functions for dedicated information do not use them by default, inherited property with the same label usually does not describe the metric itself, e.g. a property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. +Entries which do not adhere to the conventions are skipped: metrics which are no objects or +whose field named by the metric key is no string, and value dimensions which are no objects or +whose column is no string. ``"value_dimensions"`` and ``"value_dimensions_shared"`` which are no +lists count as empty, a metric key which is no string as missing, i.e. ``"id"`` is used. + How to query nested information? ================================ diff --git a/src/hdhelpers/metadata/_current_convention.py b/src/hdhelpers/metadata/_current_convention.py new file mode 100644 index 0000000..a6434c6 --- /dev/null +++ b/src/hdhelpers/metadata/_current_convention.py @@ -0,0 +1,210 @@ +"""Extraction of metadata following the current metadata convention + +The metadata is gathered in a single pass in plain Python: the object of each metric with its +value dimensions by column, and the value dimensions shared by all metrics +("value_dimensions_shared"). The fallback chains are resolved on these objects, applying the +spec of the requested information only to the objects of the levels actually needed. Field +names and dotted paths of them are looked up without glom, since evaluating glom specs costs a +lot of time per object, which adds up for frames with many metrics. + +Entries not adhering to the conventions are skipped: metrics which are no objects or whose +field named by the metric key is no string, and value dimensions which are no objects or whose +column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count +as empty, a metric key which is no string as missing, i.e. the default "id" is used. + +Older metadata structures are handled by the glom specs in _specs. +""" + +from collections import defaultdict +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any + +from glom import Coalesce, Spec, glom + +from hdhelpers.metadata._specs import INHERITED + +Getter = Callable[[Any], Any] + +_MISSING = object() +_DEFAULT_COALESCE_SKIP = Coalesce("any").skip + + +@dataclass(frozen=True) +class GatheredMetric: + metric: dict + value_dimensions: dict[str, dict] # by column + + +@dataclass(frozen=True) +class GatheredMetadata: + metrics: dict[str, GatheredMetric] # by value of the field named by the metric key + value_dimensions_shared: dict[str, dict] # by column + + +def follows_current_convention(attrs: Any) -> bool: + """Whether metadata follows the current metadata convention, i.e. has a list of metrics""" + return isinstance(attrs, dict) and isinstance(attrs.get("metrics"), list) + + +def _metric_key(attrs: dict) -> str: + dataset_metadata = attrs.get("dataset_metadata") + metric_key = dataset_metadata.get("metric_key") if isinstance(dataset_metadata, dict) else None + return metric_key if isinstance(metric_key, str) else "id" + + +def _by_column(value_dimensions: Any) -> dict[str, dict]: + """Value dimension objects by column, the last one winning, skipping malformed ones""" + if not isinstance(value_dimensions, list): + return {} + return { + value_dimension["column"]: value_dimension + for value_dimension in value_dimensions + if isinstance(value_dimension, dict) and isinstance(value_dimension.get("column"), str) + } + + +def gather(attrs: Any) -> GatheredMetadata | None: + """Gather the objects of all levels, None if not following the current convention""" + if not follows_current_convention(attrs): + return None + + metric_key = _metric_key(attrs) + return GatheredMetadata( + metrics={ + metric[metric_key]: GatheredMetric(metric, _by_column(metric.get("value_dimensions"))) + for metric in attrs["metrics"] + if isinstance(metric, dict) and isinstance(metric.get(metric_key), str) + }, + value_dimensions_shared=_by_column(attrs.get("value_dimensions_shared")), + ) + + +def _simple_path(spec: Any) -> tuple[str, ...] | None: + """The keys a spec looks up one after another, if it is just a path of dict keys + + glom splits string paths at dots, and only interprets stars besides. + """ + if isinstance(spec, str) and "*" not in spec and "\\" not in spec: + return tuple(spec.split(".")) + if ( + isinstance(spec, Coalesce) + and len(spec.subspecs) == 1 + and spec.default is None + and spec.skip is _DEFAULT_COALESCE_SKIP + ): + return _simple_path(spec.subspecs[0]) + return None + + +def getter(spec: str | Spec) -> Getter: + """Apply spec as Coalesce(spec, default=None), without glom for paths of dict keys""" + generic_spec = Coalesce(spec, default=None) + + def generic(obj: Any) -> Any: + return glom(obj, generic_spec) + + path = _simple_path(spec) + if path is None: + return generic + + def by_path(obj: Any) -> Any: + # glom accesses keys of dicts via item access only, so a missing key yields None. + # Other targets, e.g. lists accessed via indices, are left to glom. + target = obj + for key in path: + if type(target) is not dict: + return generic(obj) + if key not in target: + return None + target = target[key] + return target + + return by_path + + +def _inherited_info(metric: dict, get: Getter) -> Any: + # the spec is not applied if the metric does not provide inherited properties at all + return get(metric[INHERITED]) if INHERITED in metric else None + + +def value_dimension_info( + gathered: GatheredMetadata, specs: Sequence[str | Spec], include_inherited: bool +) -> defaultdict[str, defaultdict[str, Any]]: + """Information per value dimension per metric, falling back to further specs + + The value dimensions of a metric are its own ones, "value" and the shared ones. For each + of them, the first information which is not None is used: Each spec is applied on + + 1. the value dimension itself, + 2. only for the value dimension "value": the metric, + 3. "value_dimensions_shared" + + before the next spec is tried. If include_inherited, the properties the metric inherits + are used as last resort for the value dimension "value", again trying each spec in turn. + """ + getters = [getter(spec) for spec in specs] + shared_info_cache: list[dict[str, Any]] = [{} for _ in getters] + + def shared_info(index: int, column: str) -> Any: + cache = shared_info_cache[index] + if column not in cache: + shared = gathered.value_dimensions_shared.get(column) + cache[column] = None if shared is None else getters[index](shared) + return cache[column] + + result: defaultdict[str, defaultdict[str, Any]] = defaultdict(lambda: defaultdict(lambda: None)) + for metric, gathered_metric in gathered.metrics.items(): + info_by_column: defaultdict[str, Any] = defaultdict(lambda: None) + columns = dict.fromkeys( + [*gathered.value_dimensions_shared, "value", *gathered_metric.value_dimensions] + ) + for column in columns: + value_dimension = gathered_metric.value_dimensions.get(column) + info = None + for index, get in enumerate(getters): + info = get(value_dimension) if value_dimension is not None else None + if info is None and column == "value": + info = get(gathered_metric.metric) + if info is None: + info = shared_info(index, column) + if info is not None: + break + if info is None and include_inherited and column == "value": + for get in getters: + info = _inherited_info(gathered_metric.metric, get) + if info is not None: + break + info_by_column[column] = info + result[metric] = info_by_column + return result + + +def metric_info( + gathered: GatheredMetadata, spec: str | Spec, include_inherited: bool +) -> dict[str, Any]: + """Information per metric, the first one which is not None of + + 1. the metric itself, + 2. its value dimension "value", + 3. the value dimension "value" in "value_dimensions_shared", + 4. if include_inherited, the properties the metric inherits. + """ + get = getter(spec) + shared = gathered.value_dimensions_shared.get("value") + shared_info: Any = _MISSING + + result: dict[str, Any] = {} + for metric, gathered_metric in gathered.metrics.items(): + info = get(gathered_metric.metric) + if info is None: + value_dimension = gathered_metric.value_dimensions.get("value") + info = get(value_dimension) if value_dimension is not None else None + if info is None and shared is not None: + if shared_info is _MISSING: + shared_info = get(shared) + info = shared_info + if info is None and include_inherited: + info = _inherited_info(gathered_metric.metric, get) + result[metric] = info + return result diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index a40a563..89c5e02 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -4,6 +4,7 @@ import pandas as pd from glom import Check, Coalesce, CoalesceError, Spec, glom +import hdhelpers.metadata._current_convention as _current_convention import hdhelpers.metadata._specs as _specs @@ -59,54 +60,23 @@ def get_value_dimension_info( If no information is found, None is set as value and is the default value of the inner default dict. + For metadata following the current metadata convention, value_dim_info is only applied to + the levels needed and entries not adhering to the conventions are skipped, see + _current_convention. + For examples we refer to the corresponding unit tests (/tests/helpers/test_metadata.py). """ + gathered = _current_convention.gather(multitsframe.attrs) + if gathered is not None: + return _current_convention.value_dimension_info( + gathered, [value_dim_info], include_inherited + ) return defaultdict( lambda: defaultdict(lambda: None), - _value_dimension_info_of_attrs(multitsframe.attrs, value_dim_info, include_inherited), + glom(multitsframe.attrs, _specs.by_metric_key_by_val_dimension(value_dim_info)), ) -def _value_dimension_info_of_attrs( - attrs: Any, value_dim_info: str | Spec, include_inherited: bool = False -) -> Any: - spec = _specs.by_metric_key_by_val_dimension(value_dim_info, include_inherited) - return glom(attrs, spec) - - -def follows_current_convention(attrs: Any) -> bool: - """Whether metadata follows the current metadata convention, i.e. has a list of metrics""" - return isinstance(attrs, dict) and isinstance(attrs.get("metrics"), list) - - -def _restricted_to_metrics(attrs: dict, metrics: set) -> dict: - """The metadata with only the metric objects of the given metrics - - The information of a metric is extracted independently of the other metrics, so fallbacks - only need to be evaluated for the metrics still lacking information. Besides saving time - for frames with many metrics, this avoids applying a spec to fallback levels whose values - (e.g. null) the spec cannot handle, while the metric itself provides the information. - - Only for metadata following the current metadata convention. - """ - metric_key = extract_from_metadata(attrs, key="metric_key", default="id") - return attrs | { - "metrics": [ - metric - for metric in attrs["metrics"] - if isinstance(metric, dict) and metric.get(metric_key) in metrics - ] - } - - -def _metrics_lacking_info(info_by_metric_by_value_dimension: dict) -> set: - return { - metric - for metric, info_by_value_dimension in info_by_metric_by_value_dimension.items() - if any(info is None for info in info_by_value_dimension.values()) - } - - def value_dimension_info_with_fallbacks( data: pd.DataFrame | pd.Series, fields: tuple[str, ...], @@ -124,42 +94,10 @@ def value_dimension_info_with_fallbacks( Older metadata structures are handled via legacy_spec, keeping their established behaviour. """ - if not follows_current_convention(data.attrs): + gathered = _current_convention.gather(data.attrs) + if gathered is None: return get_value_dimension_info(data, legacy_spec, include_inherited) - - result: defaultdict[str, defaultdict[str, Any]] = defaultdict(lambda: defaultdict(lambda: None)) - attrs = data.attrs - for field in fields: - info_by_metric = _value_dimension_info_of_attrs(attrs, Coalesce(field, default=None)) - for metric, info_by_value_dimension in info_by_metric.items(): - for value_dimension, info in info_by_value_dimension.items(): - if result[metric][value_dimension] is None: - result[metric][value_dimension] = info - - # further fields only for the metrics still lacking information - metrics_lacking_info = _metrics_lacking_info(result) - if len(metrics_lacking_info) == 0: - return result - attrs = _restricted_to_metrics(data.attrs, metrics_lacking_info) - - if include_inherited: - # inherited properties only after all fields of the metadata itself and, as in - # get_value_dimension_info, only for the value dimension "value" - for field in fields: - metrics_lacking_value_info = { - metric - for metric, info_by_value_dimension in result.items() - if info_by_value_dimension["value"] is None - } - if len(metrics_lacking_value_info) == 0: - break - inherited_by_metric = glom( - _restricted_to_metrics(data.attrs, metrics_lacking_value_info), - _specs.inherited_by_metric_key(Coalesce(field, default=None)), - ) - for metric in metrics_lacking_value_info: - result[metric]["value"] = inherited_by_metric.get(metric) - return result + return _current_convention.value_dimension_info(gathered, fields, include_inherited) _SINGLE_METRIC_KEY = "single_metric" @@ -230,34 +168,21 @@ def metric_info_by_metric( For metadata following the current metadata convention, information the metric itself does not provide is taken from its value dimension "value", then from "value_dimensions_shared" and, if include_inherited, finally from the properties the metric inherits. Information - provided as null counts as missing. The fallbacks are only evaluated for metrics lacking the - information, see _restricted_to_metrics. + provided as null counts as missing. metric_info is only applied to the levels needed and + entries not adhering to the conventions are skipped, see _current_convention. Returns an empty dict if no metric metadata is present at all. Errors raised by applying metric_info, e.g. a transformation failing on null, are not caught. """ + gathered = _current_convention.gather(data.attrs) + if gathered is not None: + return _current_convention.metric_info(gathered, metric_info, include_inherited) + try: - info_by_metric = glom(data.attrs, _specs.spec_by_metric_key(metric_info)) + return glom(data.attrs, _specs.spec_by_metric_key(metric_info)) # type: ignore[no-any-return] except CoalesceError: # no metric metadata present at all return {} - if not follows_current_convention(data.attrs): - return info_by_metric - - metrics_lacking_info = {metric for metric, info in info_by_metric.items() if info is None} - if len(metrics_lacking_info) == 0: - return info_by_metric - - value_dimension_info = _value_dimension_info_of_attrs( - _restricted_to_metrics(data.attrs, metrics_lacking_info), - Coalesce(metric_info, default=None), - include_inherited, - ) - return { - metric: info if info is not None else value_dimension_info.get(metric, {}).get("value") - for metric, info in info_by_metric.items() - } - def singlets_metric_info( singletsframe: pd.DataFrame, metric_info: str | Spec, include_inherited: bool = True diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index dec0ed1..a0fdcb3 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -1,8 +1,4 @@ -from collections import defaultdict -from collections.abc import Callable -from typing import Any - -from glom import A, Check, Coalesce, Iter, Merge, S, Spec, T, Val +from glom import Check, Coalesce, Iter, Merge, Spec, T # The metadatum specs of the individual accessors, including their fallback chains. # Defined once here and shared by the MultiTSFrame and the SingleTSFrame accessors, which @@ -19,15 +15,16 @@ # The fallback chains of the display names for metadata following the current metadata # convention: Each field is looked up on all levels (value dimension, metric, shared value -# dimensions) before the next field is tried, see value_dimension_info_with_fallbacks. +# dimensions) before the next field is tried, see _current_convention.value_dimension_info. DISPLAY_NAME_FIELDS = ("display_name", "name") SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", "display_name", "name") -def by_metric_key_by_val_dimension( - metadatum_key: str | Spec, include_inherited: bool = False -) -> Spec: - """Providesglom spec that extracts a metadatum by metric by value dimension +def by_metric_key_by_val_dimension(metadatum_key: str | Spec) -> Spec: + """Provides glom spec that extracts a metadatum by metric by value dimension + + Only for older metadata structures, metadata following the current metadata convention is + handled by _current_convention. The generated glom spec returns a defaultdict of defaultdicts: {metric_key: {value_dimension_column_name: metadatum_value}} @@ -40,14 +37,9 @@ def by_metric_key_by_val_dimension( Properly falls back to "value_dimensions_shared" metadata if a value_dimension is not given for a metric if its available there. - For the current metadata convention and include_inherited, the properties the metric - inherits are used as last resort for the "value" value_dimension, i.e. after - "value_dimensions_shared". - metadatum_key can also be any glom spec. """ return Coalesce( - _spec_new_convention(metadatum_key, include_inherited), _spec_platform_convention_series_metric(metadatum_key), _spec_platform_convention_series_value_dim(metadatum_key), _spec_older_convention1_metric(metadatum_key), @@ -58,157 +50,6 @@ def by_metric_key_by_val_dimension( ) -def _spec_metric_key() -> Spec: - """Which key of the metric objects identifies a metric - - Per the metadata conventions "metric_key" is optional and defaults to "id". Note that - it therefore cannot act as the discriminator between the current and the older metadata - conventions - that role belongs to "metrics" being a list, see - _spec_actual_per_metric_per_value_dimensions. - """ - return (Coalesce("dataset_metadata.metric_key", default="id"), A.globals.metric_key) - - -def _spec_defaults_by_value_dimension(metadatum_key: str | Spec) -> Spec: - return Coalesce( - ( - Coalesce("value_dimensions_shared", default=[]), - Check(instance_of=list), - _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - "column", Coalesce(metadatum_key, default=None) - ), - ), - default={}, - ) - - -def _spec_defaults_by_metric(metadatum_key: str | Spec) -> Spec: - return Coalesce( - ( - "metrics", - Check(instance_of=list), - _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - S.globals.metric_key, - Coalesce(metadatum_key, default=None), - key_as_value=True, - ), - ), - default={}, - ) - - -def _spec_inherited_by_metric(metadatum_key: str | Spec) -> Spec: - """The metadatum from the properties each metric inherits, keyed by metric""" - return Coalesce( - ( - "metrics", - Check(instance_of=list), - _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - S.globals.metric_key, - Coalesce((INHERITED, metadatum_key), default=None), - key_as_value=True, - ), - ), - default={}, - ) - - -def inherited_by_metric_key(metadatum_key: str | Spec) -> Spec: - """Provides glom spec that extracts a metadatum from the properties each metric inherits - - Only for the current metadata convention. Returns a dict keyed by metric, which is empty if - there is no list of metrics. - """ - return ( - {"metric_key": _spec_metric_key(), "by_metric": _spec_inherited_by_metric(metadatum_key)}, - "by_metric", - ) - - -def _spec_actual_per_metric_per_value_dimensions(metadatum_key: str | Spec) -> Spec: - """Deliberately without a default: this is what discriminates the metadata conventions - - If "metrics" is absent or is not a list, this spec fails, which makes the enclosing - _spec_new_convention fail, so that by_metric_key_by_val_dimension falls through to the - older conventions. - """ - return ( - "metrics", - Check(instance_of=list), - _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - S.globals.metric_key, - ( - Coalesce("value_dimensions", default={}), - _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - "column", - Coalesce(metadatum_key, default=None), - add_keys_with_none_values=["value"], - ), - ), - key_as_value=True, - ), - ) - - -def _spec_new_convention(metadatum_key: str | Spec, include_inherited: bool) -> Spec: - return ( - { # first gather information at different locations in metadata in a dict - "metric_key": _spec_metric_key(), - "defaults_by_value_dimension": _spec_defaults_by_value_dimension(metadatum_key), - "defaults_by_metric": _spec_defaults_by_metric(metadatum_key), - "inherited_by_metric": ( - _spec_inherited_by_metric(metadatum_key) if include_inherited else Val({}) - ), - "actual_per_metric_per_value_dimensions": _spec_actual_per_metric_per_value_dimensions( - metadatum_key - ), - }, - _combine_gathered_information, - ) - - -def _first_not_none(*values: Any) -> Any: - return next((value for value in values if value is not None), None) - - -def _combine_gathered_information(gathered: dict) -> defaultdict: - """Combine the information gathered by _spec_new_convention - - For each value dimension of each metric, the first information which is not None is used, - in the following order: - 1. the value dimension itself - 2. only for the value dimension "value": the metric - 3. "value_dimensions_shared" - 4. only for the value dimension "value": the properties the metric inherits - """ - return defaultdict( - lambda: defaultdict(lambda: None, {}), - { - metric: defaultdict( - lambda: None, - { - value_dim: _first_not_none( - info, - gathered["defaults_by_metric"].get(metric) - if value_dim == "value" - else None, - gathered["defaults_by_value_dimension"].get(value_dim), - gathered["inherited_by_metric"].get(metric) - if value_dim == "value" - else None, - ) - for value_dim, info in _update_dict_and_return_it( - gathered["defaults_by_value_dimension"].copy(), info_by_val_dim - ).items() - }, - ) - for metric, info_by_val_dim in gathered[ - "actual_per_metric_per_value_dimensions" - ].items() - }, - ) - - def _spec_older_convention1_value_dim(metadatum_key: str | Spec) -> Spec: return ( "by_metric", @@ -278,23 +119,12 @@ def _spec_older_convention4(metadatum_key: str | Spec) -> Spec: def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: - """Provides glom spec that extracts a metadatum from the metric objects, keyed by metric""" + """Provides glom spec that extracts a metadatum from the metric objects, keyed by metric + + Only for older metadata structures, metadata following the current metadata convention is + handled by _current_convention. + """ return Coalesce( - ( # current metdadata convention - { - "metric_key": _spec_metric_key(), - "by_metric": ( - "metrics", - Check(instance_of=list), - _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - S.globals.metric_key, - Coalesce(metadatum_key, default=None), - key_as_value=True, - ), - ), - }, - lambda x: defaultdict(lambda: None, x["by_metric"]), - ), ( # some older, simpler metadata structure "by_metric", _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( @@ -310,62 +140,6 @@ def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: ) -def _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result( - key_spec: Spec, - value_spec: Spec, - add_keys_with_none_values: list[str] | None = None, - default_dict_func: Callable | None = None, - continuation_spec: Spec | None = None, - key_as_value: bool = False, -) -> Spec: - """Build dict from an iterable - - Spec to convert an iterable of objects to a dict using one of their fields - (or something deeper) as keys and something else as values. - - The key something and the value something can be arbitrary specs that are applyable - on each item. - - The resulting glom spec first produces a dictionary which keys being extracted from - each element of the iterable using key_spec and values using value_spec. - - If given it then proceeds on the resulting object using the continuation_spec. - - add_keys_with_none_values allows to add keys even if they do not occur - with a default value of None. - - .. doctest:: - - >>> from hdhelpers.metadata._specs import _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result - >>> from glom import glom - >>> data = { - ... "some": [ - ... {"id": "42", "name": "some_name", "sub": {"unit": "l"}}, - ... {"id": "53", "name": "another", "sub": {"unit": "m"}},] - ... } - >>> result = glom(data, ("some", _build_dict_from_iterable_from_key_and_subspec_and_then_proceed_on_result("id", "sub.unit"))) - >>> result["42"] - 'l' - >>> result["53"] - 'm' - """ - - if add_keys_with_none_values is None: - add_keys_with_none_values = [] - - start_dict = dict.fromkeys(add_keys_with_none_values, None) - - if default_dict_func is not None: - start_dict = defaultdict(default_dict_func, start_dict) - - return ( - [{"key": key_spec if not key_as_value else T[key_spec], "value": value_spec}], - [lambda x: (x.get("key"), x.get("value"))], - dict, - lambda x: _update_dict_and_return_it(start_dict.copy(), x), - ) + ((continuation_spec,) if continuation_spec is not None else ()) - - def _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( deeper_glom_spec: Spec, add_keys_with_none_values: list[str] | None = None ) -> Spec: diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 106d53d..a765695 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -17,12 +17,12 @@ from glom import Coalesce, Spec, glom import hdhelpers.metadata._specs as _specs +from hdhelpers.metadata._current_convention import follows_current_convention from hdhelpers.metadata._helpers import ( check_dataframe, check_series, check_series_or_dataframe, extract_from_metadata, - follows_current_convention, get_value_dimension_info, metric_info_by_metric, singlets_metric_info, diff --git a/tests/metadata/test_metadata_current_convention.py b/tests/metadata/test_metadata_current_convention.py new file mode 100644 index 0000000..f425d86 --- /dev/null +++ b/tests/metadata/test_metadata_current_convention.py @@ -0,0 +1,179 @@ +"""Extraction of metadata following the current metadata convention in a single pass""" + +import pandas as pd +import pytest +from glom import Coalesce, Path + +import hdhelpers.metadata._current_convention as _current_convention +from hdhelpers.metadata import ( + get_display_names, + get_metric_info, + get_series_info, + get_short_display_names, + get_singlets_info, + get_singlets_metric_info, + get_singlets_units, + get_units, +) + + +def frame(attrs): + df = pd.DataFrame() + df.attrs = attrs + return df + + +def many_metrics_attrs(number): + return { + "dataset_metadata": {"metric_key": "id"}, + "metrics": [ + { + "id": f"m{i}", + "name": f"Metric {i}", + "display_name": None, + "inherited": {"site": "north"}, + "value_dimensions": [{"column": "value", "name": f"Metric {i}", "unit": "m"}], + } + for i in range(number) + ], + } + + +def test_entries_not_adhering_to_the_conventions_are_skipped(): + attrs = { + "metrics": [ + "no object", + {"name": "without metric key"}, + {"id": None, "name": "null metric key"}, + {"id": 42, "name": "metric key no string"}, + { + "id": "a", + "value_dimensions": [ + {"column": "value", "unit": "m"}, + {"unit": "without column"}, + {"column": None, "unit": "null column"}, + {"column": ["no", "string"], "unit": "column no string"}, + "no object", + ], + }, + {"id": "b", "unit": "m³/h", "value_dimensions": None}, + ], + "value_dimensions_shared": [ + {"column": "state", "unit": "shared unit"}, + {"unit": "without column"}, + "no object", + ], + } + + units = get_units(frame(attrs)) + + assert set(units) == {"a", "b"} + assert units["a"] == {"state": "shared unit", "value": "m"} + assert units["b"] == {"state": "shared unit", "value": "m³/h"} + assert set(get_metric_info(frame(attrs), "unit")) == {"a", "b"} + + +@pytest.mark.parametrize("shared", [None, "no list", {"column": "value", "unit": "m"}]) +def test_value_dimensions_shared_which_is_no_list_counts_as_empty(shared): + attrs = {"metrics": [{"id": "a"}], "value_dimensions_shared": shared} + + assert get_units(frame(attrs)) == {"a": {"value": None}} + + +@pytest.mark.parametrize("metric_key", [None, 42]) +def test_metric_key_which_is_no_string_defaults_to_id(metric_key): + attrs = { + "dataset_metadata": {"metric_key": metric_key, "single_metric": "a"}, + "metrics": [{"id": "a", "external_id": "ext", "unit": "m"}], + } + + assert get_metric_info(frame(attrs), "external_id") == {"a": "ext"} + assert get_singlets_metric_info(frame(attrs), "unit") == "m" + assert get_singlets_units(frame(attrs))["value"] == "m" + + +@pytest.mark.parametrize( + ("spec", "expected"), + [ + ("name", "n"), + ("missing", None), + ("items", None), # glom does not fall back to attributes of dicts + (Coalesce("name", default=None), "n"), + (Coalesce("display_name", "name", default=None), None), # null display_name is present + ("sensor.manufacturer", "Acme"), + ("sensor.missing", None), + ("display_name.missing", None), # path through null, left to glom + ("comments.0.content", "c"), # list index, left to glom + ("operator.name", None), # string paths are split at dots + (Path("operator.name"), "Ruhrverband"), + (("comments", ["content"]), ["c"]), + ], +) +def test_getter_applies_spec_like_glom(spec, expected): + obj = { + "name": "n", + "display_name": None, + "sensor": {"manufacturer": "Acme"}, + "operator.name": "Ruhrverband", + "comments": [{"content": "c"}], + } + + assert _current_convention.getter(spec)(obj) == expected + + +def test_spec_is_only_applied_to_levels_needed(): + attrs = { + "metrics": [ + {"id": "a", "name": "Sensor", "value_dimensions": [{"column": "value", "name": None}]} + ], + "value_dimensions_shared": [{"column": "value", "name": None}], + } + + assert get_metric_info(frame(attrs), ("name", str.upper))["a"] == "SENSOR" + + +def test_unused_inherited_properties_are_not_evaluated(): + attrs = { + "dataset_metadata": {"single_metric": "a"}, + "metrics": [ + { + "id": "a", + "name": "Sensor", + "inherited": {"name": None}, + "value_dimensions": [{"column": "value", "name": "Value"}], + } + ], + } + series = pd.Series([1.0]) + series.attrs = attrs + + assert get_series_info(series, ("name", str.upper)) == "VALUE" + assert get_singlets_info(frame(attrs), ("name", str.upper))["value"] == "VALUE" + + +def test_no_glom_evaluation_per_metric_for_field_names(monkeypatch): + """glom costs a lot of time per call, which must not add up for frames with many metrics""" + calls = [] + original_glom = _current_convention.glom + monkeypatch.setattr( + _current_convention, + "glom", + lambda *args, **kwargs: calls.append(1) or original_glom(*args, **kwargs), + ) + multitsframe = frame(many_metrics_attrs(100)) + + get_units(multitsframe) + get_display_names(multitsframe, include_inherited=True) + get_short_display_names(multitsframe, include_inherited=True) + get_metric_info(multitsframe, "unit") + + assert calls == [] + + +def test_many_metrics(): + multitsframe = frame(many_metrics_attrs(1000)) + + assert get_short_display_names(multitsframe)["m999"]["value"] == "Metric 999" + assert get_units(multitsframe)["m0"]["value"] == "m" + assert get_metric_info(multitsframe, "unit")["m500"] == "m" + assert get_metric_info(multitsframe, "site")["m1"] == "north" From bba03ecfcce5c0a7a23c1f770209ec0281c37d28 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Thu, 22 Oct 2026 12:04:32 +0200 Subject: [PATCH 08/15] add helper for default timezone with fallback behaviour --- CHANGELOG.md | 1 + docs/_sources/first_steps.rst.txt | 69 +++++++++ docs/first_steps.html | 63 ++++++++ docs/genindex.html | 6 + docs/index.html | 131 ++++++++++++++++ docs/objects.inv | Bin 525 -> 540 bytes docs/searchindex.js | 2 +- sphinx/source/first_steps.rst | 69 +++++++++ src/hdhelpers/metadata/__init__.py | 6 + src/hdhelpers/metadata/_current_convention.py | 8 + src/hdhelpers/metadata/_helpers.py | 49 +++++- src/hdhelpers/metadata/_specs.py | 4 + src/hdhelpers/metadata/helpers.py | 132 ++++++++++++++++ .../test_metadata_default_timezone.py | 146 ++++++++++++++++++ 14 files changed, 684 insertions(+), 2 deletions(-) create mode 100644 tests/metadata/test_metadata_default_timezone.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dc2895a..87d1e49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - metadata following the current convention is extracted in a single pass in plain Python instead of evaluating glom specs per level and field. The accessors are considerably faster for frames with many metrics, e.g. about 100 times for 1000 metrics when requesting field names or dotted paths. Other glom specs are still evaluated by glom, but only on the levels needed - the requested information is only looked up on the levels needed, so e.g. a spec transforming the information is no longer applied to null values of levels which are not used anyway - entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error +- new functions get_default_timezones, get_series_default_timezone and get_singlets_default_timezone provide the default time zone of the metrics ("default_timezone" of the current metadata convention), falling back to the one of the dataset metadata. It is meant for computations, e.g. bucketing days when aggregating, not for displaying timestamps, which is what the time zone of the plot target settings is for. New section "Which time zone to compute in?" in the documentation - documentation: the information requested from get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info is interpreted as a glom spec. New section "How to query nested information?" with examples for dotted paths into sub dictionaries, other glom specs and keys containing dots ## [0.0.9] - 2026-09-10 diff --git a/docs/_sources/first_steps.rst.txt b/docs/_sources/first_steps.rst.txt index 13aded9..24dab44 100644 --- a/docs/_sources/first_steps.rst.txt +++ b/docs/_sources/first_steps.rst.txt @@ -164,6 +164,75 @@ after retrieving it. .. _glom: https://glom.readthedocs.io/ +Which time zone to compute in? +============================== + +Metadata may provide a ``default_timezone`` for each metric and for the dataset metadata: the +time zone the timeseries should be evaluated in by default. It belongs to the data, e.g. it is +the time zone of the plant a sensor belongs to, and it is meant for **computations**: +Components evaluating timeseries in calendar intervals should respect it. E.g. a component +aggregating daily values should bucket the days from midnight to midnight of this time zone, +so its results depend on it. + +It is **not** meant for displaying timestamps. That is what the time zone of the plot target +settings is for, which the hetida platform provides for the view, e.g. the time zone of a +dashboard or of the user looking at a plot. ``modify_timezone`` uses it if no ``to_timezone`` +is given. Both may differ: A report on a plant in Germany may aggregate days in +``"Europe/Berlin"`` and nevertheless show its timestamps to a viewer in New York in +``"America/New_York"``. + +``get_series_default_timezone``, ``get_singlets_default_timezone`` and +``get_default_timezones`` (keyed by metric) provide the default time zone of a metric. If the +metric does not provide one, they fall back to the one of the dataset metadata, which applies +to all metrics without their own. If neither provides one, they return None. The time zone is +an IANA time zone name like ``"Europe/Berlin"``, which respects daylight saving time, or a +fixed UTC offset like ``"+02:00"``. pandas accepts both. + +.. code-block:: python + + from hdhelpers.helpers import modify_timezone + from hdhelpers.metadata import get_series_default_timezone + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + # compute: each day starts at midnight of the default time zone of the data, + # e.g. at midnight in "Europe/Berlin", and has 23 or 25 hours when switching + # from or to daylight saving time + timezone = get_series_default_timezone(series) or "UTC" + daily_sums = modify_timezone(series, to_timezone=timezone).resample("1D").sum() + + # view: show the timestamps in the time zone of the plot target settings + daily_sums = modify_timezone(daily_sums) + + ... + +Note the explicit fallback ``or "UTC"`` in the example: Without it, ``modify_timezone`` would +use the time zone of the plot target settings if the metadata provide no default time zone, +so the results of the computation would depend on who looks at them. + +For a MULTITSFRAME, each metric may have its own default time zone: + +.. code-block:: python + + from hdhelpers.metadata import get_default_timezones + + def main(*, multitsframe): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + timezones = get_default_timezones(multitsframe) + daily_sums = { + metric: group.set_index("timestamp")["value"] + .tz_convert(timezones[metric] or "UTC") + .resample("1D") + .sum() + for metric, group in multitsframe.groupby("metric") + } + + ... + How to use hdhelpers for plotting? (tbd) ======================================== diff --git a/docs/first_steps.html b/docs/first_steps.html index 93757ab..a6623f6 100644 --- a/docs/first_steps.html +++ b/docs/first_steps.html @@ -184,6 +184,68 @@

    How to query nested information? +

    Which time zone to compute in?

    +

    Metadata may provide a default_timezone for each metric and for the dataset metadata: the +time zone the timeseries should be evaluated in by default. It belongs to the data, e.g. it is +the time zone of the plant a sensor belongs to, and it is meant for computations: +Components evaluating timeseries in calendar intervals should respect it. E.g. a component +aggregating daily values should bucket the days from midnight to midnight of this time zone, +so its results depend on it.

    +

    It is not meant for displaying timestamps. That is what the time zone of the plot target +settings is for, which the hetida platform provides for the view, e.g. the time zone of a +dashboard or of the user looking at a plot. modify_timezone uses it if no to_timezone +is given. Both may differ: A report on a plant in Germany may aggregate days in +"Europe/Berlin" and nevertheless show its timestamps to a viewer in New York in +"America/New_York".

    +

    get_series_default_timezone, get_singlets_default_timezone and +get_default_timezones (keyed by metric) provide the default time zone of a metric. If the +metric does not provide one, they fall back to the one of the dataset metadata, which applies +to all metrics without their own. If neither provides one, they return None. The time zone is +an IANA time zone name like "Europe/Berlin", which respects daylight saving time, or a +fixed UTC offset like "+02:00". pandas accepts both.

    +
    from hdhelpers.helpers import modify_timezone
+from hdhelpers.metadata import get_series_default_timezone
+
+def main(*, series):
+    # entrypoint function for this component
+    # ***** DO NOT EDIT LINES ABOVE *****
+
+    # compute: each day starts at midnight of the default time zone of the data,
+    # e.g. at midnight in "Europe/Berlin", and has 23 or 25 hours when switching
+    # from or to daylight saving time
+    timezone = get_series_default_timezone(series) or "UTC"
+    daily_sums = modify_timezone(series, to_timezone=timezone).resample("1D").sum()
+
+    # view: show the timestamps in the time zone of the plot target settings
+    daily_sums = modify_timezone(daily_sums)
+
+    ...
+
    +
    +

    Note the explicit fallback or "UTC" in the example: Without it, modify_timezone would +use the time zone of the plot target settings if the metadata provide no default time zone, +so the results of the computation would depend on who looks at them.

    +

    For a MULTITSFRAME, each metric may have its own default time zone:

    +
    from hdhelpers.metadata import get_default_timezones
+
+def main(*, multitsframe):
+    # entrypoint function for this component
+    # ***** DO NOT EDIT LINES ABOVE *****
+
+    timezones = get_default_timezones(multitsframe)
+    daily_sums = {
+        metric: group.set_index("timestamp")["value"]
+        .tz_convert(timezones[metric] or "UTC")
+        .resample("1D")
+        .sum()
+        for metric, group in multitsframe.groupby("metric")
+    }
+
+    ...
+
    +
    +

    How to use hdhelpers for plotting? (tbd)

    Let’s say we want to plot the same timeseries above using hdhelpers functionalities. @@ -235,6 +297,7 @@

    Table of Contents

  • How to get metadata with hdhelpers?
  • Where does the information come from?
  • How to query nested information?
    • +
  • Which time zone to compute in?
  • How to use hdhelpers for plotting? (tbd)
    • diff --git a/docs/genindex.html b/docs/genindex.html index d69b792..ce8be7b 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -50,6 +50,8 @@

    Index

    G

    @@ -74,6 +75,54 @@

    Functions +
    +hdhelpers.metadata.get_default_timezones(multitsframe)
    +

    Gets the default time zone of each metric of a MultiTSFrame

    +

    The default time zone is meant for computations: It is the time zone the timeseries should +be evaluated in by default, e.g. for bucketing days when aggregating, so that each day +starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of +the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the +time zone of the plot target settings is for, which e.g. hdhelpers.helpers.modify_timezone() +uses by default. See “Which time zone to compute in?” in the first steps.

    +

    The time zone is an IANA time zone name like “Europe/Berlin” or a fixed UTC offset like +“+02:00”, both accepted by pandas, e.g. by tz_convert.

    +

    The default time zone of a metric falls back to the one of the dataset metadata, which +applies to all metrics without their own. As specified by the metadata conventions, it does +not fall back to value dimensions or inherited properties.

    +
    +
    Parameters:
    +

    multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

    +
    +
    Returns:
    +

    Dictionary, where keys are the entries of the metrics metadata specified via “metric_key” in “dataset_metadata” and values are their default time zones. Defaults to the default time zone of the dataset metadata, also for metrics without metadata, and to None if that is missing as well.

    +
    +
    Return type:
    +

    defaultdict[str, str | None]

    +
    +
    Raises:
    +

    TypeError – If multitsframe is not a DataFrame.

    +
    +
    +

    Code example:

    +
    >>> multitsframe.attrs = {
+...     "dataset_metadata": {"metric_key": "id", "default_timezone": "Europe/Berlin"},
+...     "metrics": [
+...         {"id": "new_york_sensor", "default_timezone": "America/New_York"},
+...         {"id": "berlin_sensor"},
+...     ],
+... }
+>>> timezones = get_default_timezones(multitsframe)
+>>> timezones["new_york_sensor"]
+'America/New_York'
+>>> timezones["berlin_sensor"]
+'Europe/Berlin'
+>>> timezones["metric_without_metadata"]
+'Europe/Berlin'
+
    +
    +
    +
    hdhelpers.metadata.get_display_names(multitsframe, *, include_inherited=False)
    @@ -323,6 +372,47 @@

    Functions +
    +hdhelpers.metadata.get_series_default_timezone(series)
    +

    Gets the default time zone of a Series

    +

    The default time zone is meant for computations: It is the time zone the timeseries should +be evaluated in by default, e.g. for bucketing days when aggregating, so that each day +starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of +the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the +time zone of the plot target settings is for, which e.g. hdhelpers.helpers.modify_timezone() +uses by default. See “Which time zone to compute in?” in the first steps.

    +

    The time zone is an IANA time zone name like “Europe/Berlin” or a fixed UTC offset like +“+02:00”, both accepted by pandas, e.g. by tz_convert.

    +

    The default time zone of the metric falls back to the one of the dataset metadata.

    +
    +
    Parameters:
    +

    series (pd.Series) – Series with metadata following the convention.

    +
    +
    Returns:
    +

    The default time zone, or None if neither the metric nor the dataset metadata provide one.

    +
    +
    Return type:
    +

    str | None

    +
    +
    Raises:
    +

    TypeError – If series is not a Series.

    +
    +
    +

    Code example:

    +
    >>> series.attrs = {
+...     "dataset_metadata": {"single_metric": "water.north.level", "metric_key": "external_id"},
+...     "metrics": [{"external_id": "water.north.level", "default_timezone": "Europe/Berlin"}],
+... }
+>>> get_series_default_timezone(series)
+'Europe/Berlin'
+>>> series.attrs = {}
+>>> get_series_default_timezone(series) is None
+True
+
    +
    +

    +
    hdhelpers.metadata.get_series_display_name(series, *, include_inherited=False)
    @@ -595,6 +685,44 @@

    Functions +
    +hdhelpers.metadata.get_singlets_default_timezone(singletsframe)
    +

    Gets the default time zone of the single metric of a SingleTSFrame

    +

    The default time zone is meant for computations: It is the time zone the timeseries should +be evaluated in by default, e.g. for bucketing days when aggregating, so that each day +starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of +the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the +time zone of the plot target settings is for, which e.g. hdhelpers.helpers.modify_timezone() +uses by default. See “Which time zone to compute in?” in the first steps.

    +

    The time zone is an IANA time zone name like “Europe/Berlin” or a fixed UTC offset like +“+02:00”, both accepted by pandas, e.g. by tz_convert.

    +

    The default time zone of the metric falls back to the one of the dataset metadata.

    +
    +
    Parameters:
    +

    singletsframe (pd.DataFrame) – STSF with metadata following the convention.

    +
    +
    Returns:
    +

    The default time zone, or None if neither the metric nor the dataset metadata provide one.

    +
    +
    Return type:
    +

    str | None

    +
    +
    Raises:
    +

    TypeError – If singletsframe is not a DataFrame.

    +
    +
    +

    Code example:

    +
    >>> dataframe.attrs = {
+...     "dataset_metadata": {"single_metric": "abc.temp", "default_timezone": "+02:00"},
+...     "metrics": [{"id": "abc.temp", "default_timezone": None}],
+... }
+>>> get_singlets_default_timezone(dataframe)
+'+02:00'
+
    +
    +

    +
    hdhelpers.metadata.get_singlets_display_names(singletsframe, *, include_inherited=False)
    @@ -1192,11 +1320,13 @@

    Table of Contents

  • Further Information
  • Functions
    • metadata
        +
      • get_default_timezones()
      • get_display_names()
      • get_measurements()
      • get_metric_info()
      • get_names()
      • get_queried_interval()
        • +
      • get_series_default_timezone()
      • get_series_display_name()
      • get_series_info()
      • get_series_measurement()
        • @@ -1204,6 +1334,7 @@

        Table of Contents

      • get_series_short_display_name()
      • get_series_unit()
      • get_short_display_names()
        • +
      • get_singlets_default_timezone()
      • get_singlets_display_names()
      • get_singlets_info()
      • get_singlets_measurements()
        • diff --git a/docs/objects.inv b/docs/objects.inv index 4812edfa968739d176c2d42aaa3ae6e907bf379e..3fa5d63a4fb0da27092f66224ce6944cab09352a 100644 GIT binary patch delta 430 zcmV;f0a5;q1e^qrcz>3`PJ}QJhVMK@6TL2Rub$QD!DJI3fYf$?#?oSEFyhl&C?K2F z2!&J0wEs6VooP!cOVBkqOyN^NP0AW%b$eS`*=S%lOc@EEX-Sp73bw;FJ0#^Vvh=Yx z-H(P2GtIK&ob)H+$zYS`N+>XN(w0b>;*Jwv%o@g3DU4$Qlz(X}xEFbO-WajaTk5eY zZ!=OAd|9(tUIB4AWVbllQPhVkA?J;z$uq8X9ViNG&D+fIDw0qE$HqbB%N7eDr$nYq zZlm<^p}#la6iDxyz60U2Wa&N zWWs9e*1Am52g?g%k$e@Ygyo0#3?aK?BHsQ{kmRkjf*p=*X71Orht9xy zLsEe0DZN1RSJ0hgw1)Lh-DmQ%Y~apTOkpS855Xm`ZYK}pL=ZfOIt21;_8-})rP619 YrbggSl;^@=3!0ZOf8qrs->X}r(cJdYyZ`_I delta 415 zcmV;Q0bu@|1dRlccz>16PJ}QJ$L~Bv6TL#*>z>uP2a`>F08-ll8cU0vVYB%3mai4E z8lmi|WM=-q&PPiNS%9vuvEpi=*NfS@P1E zo<>89xp>)bO8OIVFj!}q5(*3*wIxy}f8@j$vxaq53FDdrrGMHA?nPdn)kZAbE%8{D zH7ThQzN}d+FM+rms#~1xDC*Lc3MSXf6lYf5cYU^Q21gq;-hzYz_Dw?u_c zdL(SVdK1#NJAZfq-JmZwpILi)!Vs&J)$rV``WcU+!~t4;f>c;--P$b^^vUwvSR_A1 zDq#6_n;|6mCrEt!Q*x#p`t}l3$dKe-TEXHSo0z*;_S_m+XNYnz9i z(0)cgVFPy(aY;LBAIxiB)x@Kt7rgd!=*biBKeA&(g&41VrbggS6mwy3>osq^|HyNQ Jz5$T5P?aDl#{2*P diff --git a/docs/searchindex.js b/docs/searchindex.js index 5e2605a..4bca6aa 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"A":[0,1],"DO":0,"Each":1,"For":[0,1],"How":1,"If":1,"In":[0,1],"It":1,"NOT":0,"Other":0,"Our":0,"So":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adher":0,"adit":1,"adjust":1,"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"ani":[0,1],"anoth":1,"anyway":0,"appli":0,"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":1,"bool":1,"boolean":1,"by_metr":1,"c":1,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"dark":1,"data":1,"datafram":1,"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"defaultdict":1,"defin":[0,1],"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":[0,1],"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":[0,1],"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":1,"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":1,"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"get":1,"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"handl":[0,1],"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"id":[0,1],"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":[0,1],"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"may":0,"meaning":1,"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":1,"nest":1,"never":0,"next":[0,1],"non":1,"none":[0,1],"north":0,"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":0,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":1,"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":1,"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"repres":0,"request":[0,1],"requir":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":1,"result":1,"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":1,"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"skip":0,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"surpress":1,"switch":[0,1],"t":1,"target":1,"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":1,"timezon":[0,1],"to_datetim":1,"to_timezon":1,"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":1,"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"usual":0,"utc":1,"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"vision":0,"want":0,"water":0,"way":1,"well":1,"whether":1,"whole":0,"whose":[0,1],"will":1,"win":0,"without":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"come":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"use":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"Which time zone to compute in?":[[0,"which-time-zone-to-compute-in"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_default_timezones() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_timezones",false]],"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_timezone",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_timezone",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_default_timezones"],[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_default_timezone"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_default_timezone"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"1d":0,"A":[0,1],"As":1,"Both":0,"DO":0,"Each":1,"For":[0,1],"How":1,"If":[0,1],"In":[0,1],"It":[0,1],"NOT":0,"Other":0,"Our":0,"So":0,"That":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"Which":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"accept":[0,1],"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adher":0,"adit":1,"adjust":1,"aggreg":[0,1],"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"america":[0,1],"ani":[0,1],"anoth":1,"anyway":0,"appli":[0,1],"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":[0,1],"berlin_sensor":1,"bool":1,"boolean":1,"bucket":[0,1],"by_metr":1,"c":1,"calendar":0,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"comput":1,"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"daili":0,"daily_sum":0,"dark":1,"dashboard":0,"data":[0,1],"datafram":1,"dataset":[0,1],"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"day":[0,1],"daylight":0,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"default_timezon":[0,1],"defaultdict":1,"defin":[0,1],"depend":0,"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":[0,1],"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":[0,1],"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":[0,1],"evalu":[0,1],"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":[0,1],"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"fix":[0,1],"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"germani":0,"get":1,"get_default_timezon":[0,1],"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_default_timezon":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_default_timezon":[0,1],"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"group":0,"groupbi":0,"handl":[0,1],"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"hour":0,"iana":[0,1],"id":[0,1],"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":[0,1],"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":[0,1],"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"may":0,"meaning":1,"meant":[0,1],"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"metric_without_metadata":1,"midnight":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":[0,1],"nest":1,"never":0,"nevertheless":0,"new":0,"new_york":[0,1],"new_york_sensor":1,"next":[0,1],"non":1,"none":[0,1],"north":[0,1],"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":[0,1],"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":[0,1],"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":[0,1],"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"report":0,"repres":0,"request":[0,1],"requir":0,"resampl":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":[0,1],"result":[0,1],"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"save":0,"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_index":0,"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":[0,1],"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"skip":0,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"sum":0,"surpress":1,"switch":[0,1],"t":1,"target":[0,1],"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":[0,1],"timezon":[0,1],"to_datetim":1,"to_timezon":[0,1],"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":[0,1],"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"user":0,"usual":0,"utc":[0,1],"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"view":0,"viewer":0,"vision":0,"want":0,"water":[0,1],"way":1,"well":1,"whether":1,"whole":0,"whose":[0,1],"will":1,"win":0,"without":[0,1],"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"york":0,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"Which":0,"come":0,"comput":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"time":0,"use":0,"zone":0}}) \ No newline at end of file diff --git a/sphinx/source/first_steps.rst b/sphinx/source/first_steps.rst index 13aded9..24dab44 100644 --- a/sphinx/source/first_steps.rst +++ b/sphinx/source/first_steps.rst @@ -164,6 +164,75 @@ after retrieving it. .. _glom: https://glom.readthedocs.io/ +Which time zone to compute in? +============================== + +Metadata may provide a ``default_timezone`` for each metric and for the dataset metadata: the +time zone the timeseries should be evaluated in by default. It belongs to the data, e.g. it is +the time zone of the plant a sensor belongs to, and it is meant for **computations**: +Components evaluating timeseries in calendar intervals should respect it. E.g. a component +aggregating daily values should bucket the days from midnight to midnight of this time zone, +so its results depend on it. + +It is **not** meant for displaying timestamps. That is what the time zone of the plot target +settings is for, which the hetida platform provides for the view, e.g. the time zone of a +dashboard or of the user looking at a plot. ``modify_timezone`` uses it if no ``to_timezone`` +is given. Both may differ: A report on a plant in Germany may aggregate days in +``"Europe/Berlin"`` and nevertheless show its timestamps to a viewer in New York in +``"America/New_York"``. + +``get_series_default_timezone``, ``get_singlets_default_timezone`` and +``get_default_timezones`` (keyed by metric) provide the default time zone of a metric. If the +metric does not provide one, they fall back to the one of the dataset metadata, which applies +to all metrics without their own. If neither provides one, they return None. The time zone is +an IANA time zone name like ``"Europe/Berlin"``, which respects daylight saving time, or a +fixed UTC offset like ``"+02:00"``. pandas accepts both. + +.. code-block:: python + + from hdhelpers.helpers import modify_timezone + from hdhelpers.metadata import get_series_default_timezone + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + # compute: each day starts at midnight of the default time zone of the data, + # e.g. at midnight in "Europe/Berlin", and has 23 or 25 hours when switching + # from or to daylight saving time + timezone = get_series_default_timezone(series) or "UTC" + daily_sums = modify_timezone(series, to_timezone=timezone).resample("1D").sum() + + # view: show the timestamps in the time zone of the plot target settings + daily_sums = modify_timezone(daily_sums) + + ... + +Note the explicit fallback ``or "UTC"`` in the example: Without it, ``modify_timezone`` would +use the time zone of the plot target settings if the metadata provide no default time zone, +so the results of the computation would depend on who looks at them. + +For a MULTITSFRAME, each metric may have its own default time zone: + +.. code-block:: python + + from hdhelpers.metadata import get_default_timezones + + def main(*, multitsframe): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + timezones = get_default_timezones(multitsframe) + daily_sums = { + metric: group.set_index("timestamp")["value"] + .tz_convert(timezones[metric] or "UTC") + .resample("1D") + .sum() + for metric, group in multitsframe.groupby("metric") + } + + ... + How to use hdhelpers for plotting? (tbd) ======================================== diff --git a/src/hdhelpers/metadata/__init__.py b/src/hdhelpers/metadata/__init__.py index 3b0a1b4..cc48e5d 100644 --- a/src/hdhelpers/metadata/__init__.py +++ b/src/hdhelpers/metadata/__init__.py @@ -3,11 +3,13 @@ extract the requested information.""" from hdhelpers.metadata.helpers import ( + get_default_timezones, get_display_names, get_measurements, get_metric_info, get_names, get_queried_interval, + get_series_default_timezone, get_series_display_name, get_series_info, get_series_measurement, @@ -15,6 +17,7 @@ get_series_short_display_name, get_series_unit, get_short_display_names, + get_singlets_default_timezone, get_singlets_display_names, get_singlets_info, get_singlets_measurements, @@ -46,4 +49,7 @@ "get_singlets_measurements", "get_singlets_metric_info", "get_singlets_units", + "get_default_timezones", + "get_series_default_timezone", + "get_singlets_default_timezone", ] diff --git a/src/hdhelpers/metadata/_current_convention.py b/src/hdhelpers/metadata/_current_convention.py index a6434c6..6531c9b 100644 --- a/src/hdhelpers/metadata/_current_convention.py +++ b/src/hdhelpers/metadata/_current_convention.py @@ -208,3 +208,11 @@ def metric_info( info = _inherited_info(gathered_metric.metric, get) result[metric] = info return result + + +def own_metric_info(gathered: GatheredMetadata, key: str) -> dict[str, Any]: + """A field of each metric itself, without falling back to other levels""" + return { + metric: gathered_metric.metric.get(key) + for metric, gathered_metric in gathered.metrics.items() + } diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 89c5e02..6176def 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any +from typing import Any, cast import pandas as pd from glom import Check, Coalesce, CoalesceError, Spec, glom @@ -199,3 +199,50 @@ def singlets_metric_info( singletsframe.attrs, empty=None, ) + + +def _string_or_none(value: Any) -> str | None: + return value if isinstance(value, str) and value != "" else None + + +def _dataset_default_timezone(attrs: Any) -> str | None: + return _string_or_none(extract_from_metadata(attrs, key=_specs.DEFAULT_TIMEZONE)) + + +def default_timezone_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str | None]: + """The default time zone of each metric, falling back to the one of the dataset metadata + + As specified by the metadata conventions, the default time zone of a metric only falls back + to the one of the dataset metadata, not to its value dimensions or inherited properties. + The default of the returned defaultdict is the one of the dataset metadata, since it applies + to all metrics without their own. Values which are no (non-empty) strings count as missing. + """ + dataset_timezone = _dataset_default_timezone(data.attrs) + gathered = _current_convention.gather(data.attrs) + own_timezones = ( + {} + if gathered is None + else _current_convention.own_metric_info(gathered, _specs.DEFAULT_TIMEZONE) + ) + return defaultdict( + lambda: dataset_timezone, + { + metric: _string_or_none(timezone) or dataset_timezone + for metric, timezone in own_timezones.items() + }, + ) + + +def single_metric_default_timezone(data: pd.DataFrame | pd.Series) -> str | None: + """The default time zone of the single metric of a Series or SingleTSFrame + + Falls back to the one of the dataset metadata, also if the single metric is not found. + """ + return cast( + str | None, + _select_single_metric( + default_timezone_by_metric(data), + data.attrs, + empty=_dataset_default_timezone(data.attrs), + ), + ) diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index a0fdcb3..9933c58 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -13,6 +13,10 @@ # higher up in an asset hierarchy, e.g. the latitude of the plant a sensor belongs to. INHERITED = "inherited" +# Field of a metric and of the dataset metadata: the time zone the timeseries should be +# evaluated in by default, e.g. for bucketing days when aggregating. +DEFAULT_TIMEZONE = "default_timezone" + # The fallback chains of the display names for metadata following the current metadata # convention: Each field is looked up on all levels (value dimension, metric, shared value # dimensions) before the next field is tried, see _current_convention.value_dimension_info. diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index a765695..11653fe 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -22,11 +22,13 @@ check_dataframe, check_series, check_series_or_dataframe, + default_timezone_by_metric, extract_from_metadata, get_value_dimension_info, metric_info_by_metric, singlets_metric_info, singlets_value_dimension_info, + single_metric_default_timezone, singlets_value_dimension_info_with_fallbacks, spec_not_none, value_dimension_info_with_fallbacks, @@ -1033,3 +1035,133 @@ def get_queried_interval( formatted_end = datetime.datetime.fromisoformat(end) if end else None return formatted_start, formatted_end + + +def get_default_timezones(multitsframe: pd.DataFrame) -> defaultdict[str, str | None]: + """Gets the default time zone of each metric of a MultiTSFrame + + The default time zone is meant for computations: It is the time zone the timeseries should + be evaluated in by default, e.g. for bucketing days when aggregating, so that each day + starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of + the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the + time zone of the plot target settings is for, which e.g. :func:`hdhelpers.helpers.modify_timezone` + uses by default. See "Which time zone to compute in?" in the first steps. + + The time zone is an IANA time zone name like "Europe/Berlin" or a fixed UTC offset like + "+02:00", both accepted by pandas, e.g. by ``tz_convert``. + + The default time zone of a metric falls back to the one of the dataset metadata, which + applies to all metrics without their own. As specified by the metadata conventions, it does + not fall back to value dimensions or inherited properties. + + Args: + multitsframe (pd.DataFrame): MTSF with metadata following the convention. + + Returns: + defaultdict[str, str | None]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are their default time zones. Defaults to the default time zone of the dataset metadata, also for metrics without metadata, and to None if that is missing as well. + + Raises: + TypeError: If `multitsframe` is not a DataFrame. + + Code example: + + .. doctest:: metadata.get_default_timezones + + >>> multitsframe.attrs = { + ... "dataset_metadata": {"metric_key": "id", "default_timezone": "Europe/Berlin"}, + ... "metrics": [ + ... {"id": "new_york_sensor", "default_timezone": "America/New_York"}, + ... {"id": "berlin_sensor"}, + ... ], + ... } + >>> timezones = get_default_timezones(multitsframe) + >>> timezones["new_york_sensor"] + 'America/New_York' + >>> timezones["berlin_sensor"] + 'Europe/Berlin' + >>> timezones["metric_without_metadata"] + 'Europe/Berlin' + """ + check_dataframe(multitsframe) + return default_timezone_by_metric(multitsframe) + + +def get_series_default_timezone(series: pd.Series) -> str | None: + """Gets the default time zone of a Series + + The default time zone is meant for computations: It is the time zone the timeseries should + be evaluated in by default, e.g. for bucketing days when aggregating, so that each day + starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of + the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the + time zone of the plot target settings is for, which e.g. :func:`hdhelpers.helpers.modify_timezone` + uses by default. See "Which time zone to compute in?" in the first steps. + + The time zone is an IANA time zone name like "Europe/Berlin" or a fixed UTC offset like + "+02:00", both accepted by pandas, e.g. by ``tz_convert``. + + The default time zone of the metric falls back to the one of the dataset metadata. + + Args: + series (pd.Series): Series with metadata following the convention. + + Returns: + str | None: The default time zone, or None if neither the metric nor the dataset metadata provide one. + + Raises: + TypeError: If `series` is not a Series. + + Code example: + + .. doctest:: metadata.get_series_default_timezone + + >>> series.attrs = { + ... "dataset_metadata": {"single_metric": "water.north.level", "metric_key": "external_id"}, + ... "metrics": [{"external_id": "water.north.level", "default_timezone": "Europe/Berlin"}], + ... } + >>> get_series_default_timezone(series) + 'Europe/Berlin' + >>> series.attrs = {} + >>> get_series_default_timezone(series) is None + True + """ + check_series(series) + return single_metric_default_timezone(series) + + +def get_singlets_default_timezone(singletsframe: pd.DataFrame) -> str | None: + """Gets the default time zone of the single metric of a SingleTSFrame + + The default time zone is meant for computations: It is the time zone the timeseries should + be evaluated in by default, e.g. for bucketing days when aggregating, so that each day + starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of + the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the + time zone of the plot target settings is for, which e.g. :func:`hdhelpers.helpers.modify_timezone` + uses by default. See "Which time zone to compute in?" in the first steps. + + The time zone is an IANA time zone name like "Europe/Berlin" or a fixed UTC offset like + "+02:00", both accepted by pandas, e.g. by ``tz_convert``. + + The default time zone of the metric falls back to the one of the dataset metadata. + + Args: + singletsframe (pd.DataFrame): STSF with metadata following the convention. + + Returns: + str | None: The default time zone, or None if neither the metric nor the dataset metadata provide one. + + Raises: + TypeError: If `singletsframe` is not a DataFrame. + + Code example: + + .. doctest:: metadata.get_singlets_default_timezone + + >>> dataframe.attrs = { + ... "dataset_metadata": {"single_metric": "abc.temp", "default_timezone": "+02:00"}, + ... "metrics": [{"id": "abc.temp", "default_timezone": None}], + ... } + >>> get_singlets_default_timezone(dataframe) + '+02:00' + """ + check_dataframe(singletsframe) + return single_metric_default_timezone(singletsframe) diff --git a/tests/metadata/test_metadata_default_timezone.py b/tests/metadata/test_metadata_default_timezone.py new file mode 100644 index 0000000..541f5e5 --- /dev/null +++ b/tests/metadata/test_metadata_default_timezone.py @@ -0,0 +1,146 @@ +"""Default time zone of the metrics, falling back to the one of the dataset metadata""" + +import pandas as pd +import pytest + +from hdhelpers.helpers import modify_timezone +from hdhelpers.metadata import ( + get_default_timezones, + get_series_default_timezone, + get_singlets_default_timezone, +) + + +def frame(attrs): + df = pd.DataFrame() + df.attrs = attrs + return df + + +def series(attrs, values=(1.0,), timestamps=("2026-03-29T12:00:00Z",)): + ser = pd.Series(list(values), index=pd.to_datetime(list(timestamps))) + ser.attrs = attrs + return ser + + +multitsframe_attrs = { + "dataset_metadata": {"metric_key": "external_id", "default_timezone": "Europe/Berlin"}, + "metrics": [ + {"external_id": "new_york", "default_timezone": "America/New_York"}, + {"external_id": "fixed_offset", "default_timezone": "+02:00"}, + {"external_id": "without"}, + {"external_id": "null", "default_timezone": None}, + {"external_id": "no_string", "default_timezone": 2}, + {"external_id": "empty", "default_timezone": ""}, + ], +} + + +def test_default_timezones_of_metrics_fall_back_to_dataset_metadata(): + timezones = get_default_timezones(frame(multitsframe_attrs)) + + assert timezones == { + "new_york": "America/New_York", + "fixed_offset": "+02:00", + "without": "Europe/Berlin", + "null": "Europe/Berlin", + "no_string": "Europe/Berlin", + "empty": "Europe/Berlin", + } + # the default time zone of the dataset metadata applies to all metrics without their own + assert timezones["metric without metadata"] == "Europe/Berlin" + + +def test_default_timezones_without_dataset_default_timezone(): + attrs = {"metrics": [{"id": "a", "default_timezone": "UTC"}, {"id": "b"}]} + + timezones = get_default_timezones(frame(attrs)) + + assert timezones == {"a": "UTC", "b": None} + assert timezones["metric without metadata"] is None + + +def test_default_timezone_does_not_fall_back_to_other_levels(): + """The conventions specify only the dataset metadata as fallback""" + attrs = { + "dataset_metadata": {"single_metric": "a"}, + "metrics": [ + { + "id": "a", + "inherited": {"default_timezone": "Asia/Tokyo"}, + "value_dimensions": [{"column": "value", "default_timezone": "Asia/Tokyo"}], + } + ], + "value_dimensions_shared": [{"column": "value", "default_timezone": "Asia/Tokyo"}], + } + + assert get_default_timezones(frame(attrs))["a"] is None + assert get_series_default_timezone(series(attrs)) is None + assert get_singlets_default_timezone(frame(attrs)) is None + + +@pytest.mark.parametrize( + ("attrs", "expected"), + [ + ( + { + "dataset_metadata": {"single_metric": "a", "default_timezone": "Europe/Berlin"}, + "metrics": [{"id": "a", "default_timezone": "America/New_York"}], + }, + "America/New_York", + ), + ( + { + "dataset_metadata": {"single_metric": "a", "default_timezone": "Europe/Berlin"}, + "metrics": [{"id": "a", "default_timezone": None}], + }, + "Europe/Berlin", + ), + # without single_metric the only metric is used + ({"metrics": [{"id": "a", "default_timezone": "UTC"}]}, "UTC"), + # without metric metadata the dataset metadata still applies + ({"dataset_metadata": {"default_timezone": "Europe/Berlin"}}, "Europe/Berlin"), + ( + {"dataset_metadata": {"default_timezone": "Europe/Berlin"}, "metrics": []}, + "Europe/Berlin", + ), + ({}, None), + # older metadata structures do not provide default time zones of the metrics + ({"by_metric": {"series": {"metric": {"default_timezone": "UTC"}}}}, None), + ], +) +def test_default_timezone_of_single_metric(attrs, expected): + assert get_series_default_timezone(series(attrs)) == expected + assert get_singlets_default_timezone(frame(attrs)) == expected + + +def test_wrong_types_raise(): + with pytest.raises(TypeError): + get_default_timezones(series({})) + with pytest.raises(TypeError): + get_series_default_timezone(frame({})) + with pytest.raises(TypeError): + get_singlets_default_timezone(series({})) + + +def test_aggregating_days_in_default_timezone(): + """As in the first steps: days start at midnight of the default time zone, which may + differ from UTC days and changes its UTC offset with daylight saving time""" + attrs = { + "dataset_metadata": {"single_metric": "a"}, + "metrics": [{"id": "a", "default_timezone": "Europe/Berlin"}], + } + # 2026-03-29 is the day daylight saving time starts in Germany, it has 23 hours + hourly = series( + attrs, + values=[1.0] * 26, + timestamps=pd.date_range("2026-03-28T23:00:00Z", periods=26, freq="1h"), + ) + + timezone = get_series_default_timezone(hourly) or "UTC" + daily_sums = modify_timezone(hourly, to_timezone=timezone).resample("1D").sum() + + assert daily_sums.to_dict() == { + pd.Timestamp("2026-03-29", tz="Europe/Berlin"): 23.0, + pd.Timestamp("2026-03-30", tz="Europe/Berlin"): 3.0, + } From 95c1a70d60835d7ddbe52b36e0d36a5e8c1a39e3 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Thu, 22 Oct 2026 14:33:41 +0200 Subject: [PATCH 09/15] do not replace an explicit single metric by another only metric If "single_metric" names a metric lacking in the metadata, the only metric present is another one. Its information, e.g. its default time zone, must not be used then; the information counts as missing instead, so the default time zone falls back to the one of the dataset metadata. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 1 + src/hdhelpers/metadata/_helpers.py | 7 ++++--- src/hdhelpers/metadata/helpers.py | 15 +++++++++------ tests/metadata/test_metadata_default_timezone.py | 15 +++++++++++++++ tests/metadata/test_metadata_fallbacks.py | 12 ++++++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87d1e49..a659b87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension - fix: get_metric_info returns None entries instead of raising if the metadata contains no metric information - the series accessors use the only metric if "single_metric" is missing in the dataset metadata +- fix: the SingleTSFrame accessors no longer use the only metric present if it is another one than the metric given by "single_metric" in the dataset metadata. The information counts as missing then - metadata following the current convention is extracted in a single pass in plain Python instead of evaluating glom specs per level and field. The accessors are considerably faster for frames with many metrics, e.g. about 100 times for 1000 metrics when requesting field names or dotted paths. Other glom specs are still evaluated by glom, but only on the levels needed - the requested information is only looked up on the levels needed, so e.g. a spec transforming the information is no longer applied to null values of levels which are not used anyway - entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 6176def..91b29fb 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -108,11 +108,12 @@ def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: The single metric is identified via _SINGLE_METRIC_KEY in "dataset_metadata". If that is missing but exactly one metric is present, that one is used, since a SingleTSFrame or - Series cannot be ambiguous in this respect. Otherwise `empty` is returned. + Series cannot be ambiguous in this respect. Otherwise `empty` is returned - also if the + identified metric is not present, since the only metric present is another one then. """ metric = extract_from_metadata(attrs, key=_SINGLE_METRIC_KEY) - if metric is not None and metric in info_by_metric: - return info_by_metric[metric] + if metric is not None: + return info_by_metric.get(metric, empty) if len(info_by_metric) == 1: return next(iter(info_by_metric.values())) diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 11653fe..c8c03f7 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -426,13 +426,16 @@ def get_series_info( def _series_value(info_by_metric: dict, series: pd.Series) -> Any: """Info of the value dimension "value" of the metric of a Series - The metric is identified via "single_metric" in "dataset_metadata", defaulting to "series" - as used by older metadata structures. If it is not found but exactly one metric is present, - that one is used, since a Series cannot be ambiguous in this respect. + The metric is identified via "single_metric" in "dataset_metadata". If that is missing, the + metric "series" as used by older metadata structures is used or else the only metric + present, since a Series cannot be ambiguous in this respect. An identified metric which is + not present yields None, since the only metric present is another one then. """ - series_metric_key = extract_from_metadata(series.attrs, key="single_metric", default="series") - if series_metric_key in info_by_metric: - return info_by_metric[series_metric_key].get("value") + series_metric_key = extract_from_metadata(series.attrs, key="single_metric") + if series_metric_key is not None: + return info_by_metric.get(series_metric_key, {}).get("value") + if "series" in info_by_metric: + return info_by_metric["series"].get("value") if len(info_by_metric) == 1: return next(iter(info_by_metric.values())).get("value") return None diff --git a/tests/metadata/test_metadata_default_timezone.py b/tests/metadata/test_metadata_default_timezone.py index 541f5e5..336349e 100644 --- a/tests/metadata/test_metadata_default_timezone.py +++ b/tests/metadata/test_metadata_default_timezone.py @@ -98,6 +98,21 @@ def test_default_timezone_does_not_fall_back_to_other_levels(): ), # without single_metric the only metric is used ({"metrics": [{"id": "a", "default_timezone": "UTC"}]}, "UTC"), + # but not if it is another one than the single metric + ( + { + "dataset_metadata": {"single_metric": "a", "default_timezone": "Europe/Berlin"}, + "metrics": [{"id": "b", "default_timezone": "America/New_York"}], + }, + "Europe/Berlin", + ), + ( + { + "dataset_metadata": {"single_metric": "a"}, + "metrics": [{"id": "b", "default_timezone": "America/New_York"}], + }, + None, + ), # without metric metadata the dataset metadata still applies ({"dataset_metadata": {"default_timezone": "Europe/Berlin"}}, "Europe/Berlin"), ( diff --git a/tests/metadata/test_metadata_fallbacks.py b/tests/metadata/test_metadata_fallbacks.py index f278125..7e98d43 100644 --- a/tests/metadata/test_metadata_fallbacks.py +++ b/tests/metadata/test_metadata_fallbacks.py @@ -209,6 +209,18 @@ def test_series_without_single_metric_uses_its_only_metric(): assert get_series_display_name(series(attrs)) == "Water-lvl-north" +def test_single_metric_is_not_replaced_by_another_only_metric(): + """If the metadata lacks the single metric, the only metric present is another one""" + attrs = deepcopy(platform_series_attrs) + attrs["dataset_metadata"]["single_metric"] = "another signal" + + assert get_series_unit(series(attrs)) is None + assert get_series_display_name(series(attrs)) is None + assert get_singlets_units(frame(attrs))["value"] is None + assert get_singlets_display_names(frame(attrs))["value"] is None + assert get_singlets_metric_info(frame(attrs), "name") is None + + @pytest.mark.parametrize( "attrs", [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}], From 7cbd8ebad7fe474b209eac6469b78ba42967b213 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Thu, 22 Oct 2026 19:05:53 +0200 Subject: [PATCH 10/15] resolve only the single metric and determine it in one place The accessors of Series and SingleTSFrames pick their single metric before resolving any information, so the information of other metrics is no longer resolved, which was slow for a Series carrying the metadata of many metrics. The single metric is determined in one place for all of them: "single_metric", which counts as missing if it is no string, or else the only metric present. The default "series" only remains for older metadata structures. get_series_unit and get_series_name use field names instead of glom specs for metadata following the current convention. get_series_display_name no longer falls back to the short display name, as documented. The display name specs for older metadata structures are derived from the field chains, and dead code is removed from _specs. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +- src/hdhelpers/metadata/_current_convention.py | 7 +- src/hdhelpers/metadata/_helpers.py | 182 +++++++++++++----- src/hdhelpers/metadata/_specs.py | 38 ++-- src/hdhelpers/metadata/helpers.py | 77 +++----- .../test_metadata_default_timezone.py | 8 + tests/metadata/test_metadata_fallbacks.py | 50 +++++ 7 files changed, 245 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a659b87..7669966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,12 @@ - get_metric_info and get_singlets_metric_info fall back to the value dimension "value" and to "value_dimensions_shared" if the metric itself does not provide the information - fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension - fix: get_metric_info returns None entries instead of raising if the metadata contains no metric information -- the series accessors use the only metric if "single_metric" is missing in the dataset metadata +- the series accessors determine their metric like the SingleTSFrame accessors: the one given by "single_metric" in the dataset metadata or, if that is missing, the only metric present. The metric named "series" is only used as default for older metadata structures +- a "single_metric" in the dataset metadata which is no string counts as missing. Before, e.g. a list let all accessors of Series and SingleTSFrames raise an error +- fix: get_series_display_name no longer falls back to the short display name, as documented and like get_display_names and get_singlets_display_names - fix: the SingleTSFrame accessors no longer use the only metric present if it is another one than the metric given by "single_metric" in the dataset metadata. The information counts as missing then - metadata following the current convention is extracted in a single pass in plain Python instead of evaluating glom specs per level and field. The accessors are considerably faster for frames with many metrics, e.g. about 100 times for 1000 metrics when requesting field names or dotted paths. Other glom specs are still evaluated by glom, but only on the levels needed +- the accessors of Series and SingleTSFrames only resolve the information of their single metric. So they are considerably faster for a Series carrying the metadata of many metrics, e.g. taken from a MultiTSFrame, and a spec is not applied to the other metrics. Also get_series_unit and get_series_name no longer evaluate glom specs for metadata following the current convention. Both are about 90 times faster for the metadata of 1000 metrics - the requested information is only looked up on the levels needed, so e.g. a spec transforming the information is no longer applied to null values of levels which are not used anyway - entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error - new functions get_default_timezones, get_series_default_timezone and get_singlets_default_timezone provide the default time zone of the metrics ("default_timezone" of the current metadata convention), falling back to the one of the dataset metadata. It is meant for computations, e.g. bucketing days when aggregating, not for displaying timestamps, which is what the time zone of the plot target settings is for. New section "Which time zone to compute in?" in the documentation diff --git a/src/hdhelpers/metadata/_current_convention.py b/src/hdhelpers/metadata/_current_convention.py index 6531c9b..3f589b5 100644 --- a/src/hdhelpers/metadata/_current_convention.py +++ b/src/hdhelpers/metadata/_current_convention.py @@ -17,7 +17,7 @@ from collections import defaultdict from collections.abc import Callable, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any from glom import Coalesce, Spec, glom @@ -80,6 +80,11 @@ def gather(attrs: Any) -> GatheredMetadata | None: ) +def restricted_to(gathered: GatheredMetadata, metric: str) -> GatheredMetadata: + """The gathered metadata of one of the metrics only, so that only its information is resolved""" + return replace(gathered, metrics={metric: gathered.metrics[metric]}) + + def _simple_path(spec: Any) -> tuple[str, ...] | None: """The keys a spec looks up one after another, if it is just a path of dict keys diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 91b29fb..37016fc 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -1,4 +1,5 @@ from collections import defaultdict +from collections.abc import Callable, Collection from typing import Any, cast import pandas as pd @@ -33,6 +34,16 @@ def extract_from_metadata(metadata: Any, key: str, default: str | None = None) - return glom(metadata, Coalesce(f"dataset_metadata.{key}", default=default)) +def _legacy_value_dimension_info( + attrs: Any, value_dim_info: str | Spec +) -> defaultdict[str, defaultdict[str, Any]]: + """get_value_dimension_info for older metadata structures""" + return defaultdict( + lambda: defaultdict(lambda: None), + glom(attrs, _specs.by_metric_key_by_val_dimension(value_dim_info)), + ) + + def get_value_dimension_info( multitsframe: pd.DataFrame | pd.Series, value_dim_info: str | Spec, @@ -71,15 +82,12 @@ def get_value_dimension_info( return _current_convention.value_dimension_info( gathered, [value_dim_info], include_inherited ) - return defaultdict( - lambda: defaultdict(lambda: None), - glom(multitsframe.attrs, _specs.by_metric_key_by_val_dimension(value_dim_info)), - ) + return _legacy_value_dimension_info(multitsframe.attrs, value_dim_info) def value_dimension_info_with_fallbacks( data: pd.DataFrame | pd.Series, - fields: tuple[str, ...], + fields: tuple[str | Spec, ...], legacy_spec: Spec, include_inherited: bool = False, ) -> defaultdict[str, defaultdict[str, Any]]: @@ -96,29 +104,70 @@ def value_dimension_info_with_fallbacks( """ gathered = _current_convention.gather(data.attrs) if gathered is None: - return get_value_dimension_info(data, legacy_spec, include_inherited) + return _legacy_value_dimension_info(data.attrs, legacy_spec) return _current_convention.value_dimension_info(gathered, fields, include_inherited) _SINGLE_METRIC_KEY = "single_metric" -def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: - """Pick the entry of the one metric of a SingleTSFrame or Series out of a by-metric mapping +def _single_metric_name(attrs: Any) -> str | None: + """The metric named by _SINGLE_METRIC_KEY in "dataset_metadata", if it is a string""" + metric = extract_from_metadata(attrs, key=_SINGLE_METRIC_KEY) + return metric if isinstance(metric, str) else None + - The single metric is identified via _SINGLE_METRIC_KEY in "dataset_metadata". If that is - missing but exactly one metric is present, that one is used, since a SingleTSFrame or - Series cannot be ambiguous in this respect. Otherwise `empty` is returned - also if the - identified metric is not present, since the only metric present is another one then. +def _single_metric(attrs: Any, metrics: Collection[str]) -> str | None: + """The single metric of a SingleTSFrame or Series among the metrics present + + The single metric is identified via _SINGLE_METRIC_KEY in "dataset_metadata", values which + are no strings count as missing. If it is missing but exactly one metric is present, that + one is used, since a SingleTSFrame or Series cannot be ambiguous in this respect. Otherwise + None is returned - also if the identified metric is not present, since the only metric + present is another one then. """ - metric = extract_from_metadata(attrs, key=_SINGLE_METRIC_KEY) + metric = _single_metric_name(attrs) if metric is not None: - return info_by_metric.get(metric, empty) + return metric if metric in metrics else None + + if len(metrics) == 1: + return next(iter(metrics)) + + return None + + +def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: + """Pick the entry of the single metric (see _single_metric) out of a by-metric mapping + + `empty` is returned if the single metric cannot be determined. + """ + metric = _single_metric(attrs, info_by_metric) + return empty if metric is None else info_by_metric[metric] - if len(info_by_metric) == 1: - return next(iter(info_by_metric.values())) - return empty +def _single_metric_info( + attrs: Any, + current: Callable[[_current_convention.GatheredMetadata], dict[str, Any]], + legacy: Callable[[], dict[str, Any]], + empty: Any, +) -> Any: + """Information of the single metric of a SingleTSFrame or Series (see _single_metric) + + For metadata following the current metadata convention, `current` gets the gathered metadata + of the single metric only, so that no information of the other metrics is resolved - of + which there may be many, e.g. for a Series taken from a MultiTSFrame. For older metadata + structures, the entry of the single metric is picked out of the result of `legacy`. + + `empty` is returned if the single metric cannot be determined. + """ + gathered = _current_convention.gather(attrs) + if gathered is None: + return _select_single_metric(legacy(), attrs, empty) + + metric = _single_metric(attrs, gathered.metrics) + if metric is None: + return empty + return current(_current_convention.restricted_to(gathered, metric))[metric] def singlets_value_dimension_info( @@ -137,16 +186,14 @@ def singlets_value_dimension_info( What is extracted per value dimension is exactly what get_value_dimension_info extracts - only the metric level is collapsed away. """ - return _select_single_metric( - get_value_dimension_info(singletsframe, value_dim_info, include_inherited), - singletsframe.attrs, - empty=defaultdict(lambda: None), + return singlets_value_dimension_info_with_fallbacks( + singletsframe, (value_dim_info,), value_dim_info, include_inherited ) def singlets_value_dimension_info_with_fallbacks( singletsframe: pd.DataFrame, - fields: tuple[str, ...], + fields: tuple[str | Spec, ...], legacy_spec: Spec, include_inherited: bool = False, ) -> defaultdict[str, Any]: @@ -154,13 +201,55 @@ def singlets_value_dimension_info_with_fallbacks( Like singlets_value_dimension_info, the result is keyed by value dimension only. """ - return _select_single_metric( - value_dimension_info_with_fallbacks(singletsframe, fields, legacy_spec, include_inherited), + return _single_metric_info( singletsframe.attrs, + lambda gathered: _current_convention.value_dimension_info( + gathered, fields, include_inherited + ), + lambda: _legacy_value_dimension_info(singletsframe.attrs, legacy_spec), empty=defaultdict(lambda: None), ) +# the metric of a Series in older metadata structures, used if "single_metric" is missing +_LEGACY_SERIES_METRIC = "series" + + +def series_value_info( + series: pd.Series, fields: tuple[str | Spec, ...], legacy_spec: Spec, include_inherited: bool +) -> Any: + """Info of the value dimension "value" of the single metric of a Series + + Each field is looked up as described in value_dimension_info_with_fallbacks. The single + metric is determined like for SingleTSFrames, see singlets_value_dimension_info. Only older + metadata structures name the metric of a Series "series", which is used if "single_metric" + is missing. + """ + + def legacy() -> dict[str, Any]: + info_by_metric = _legacy_value_dimension_info(series.attrs, legacy_spec) + if _single_metric_name(series.attrs) is None and _LEGACY_SERIES_METRIC in info_by_metric: + return {_LEGACY_SERIES_METRIC: info_by_metric[_LEGACY_SERIES_METRIC]} + return info_by_metric + + return _single_metric_info( + series.attrs, + lambda gathered: _current_convention.value_dimension_info( + gathered, fields, include_inherited + ), + legacy, + empty={}, + ).get("value") + + +def _legacy_metric_info(attrs: Any, metric_info: str | Spec) -> dict[str, Any]: + """metric_info_by_metric for older metadata structures""" + try: + return glom(attrs, _specs.spec_by_metric_key(metric_info)) # type: ignore[no-any-return] + except CoalesceError: # no metric metadata present at all + return {} + + def metric_info_by_metric( data: pd.DataFrame | pd.Series, metric_info: str | Spec, include_inherited: bool = True ) -> dict[str, Any]: @@ -178,11 +267,7 @@ def metric_info_by_metric( gathered = _current_convention.gather(data.attrs) if gathered is not None: return _current_convention.metric_info(gathered, metric_info, include_inherited) - - try: - return glom(data.attrs, _specs.spec_by_metric_key(metric_info)) # type: ignore[no-any-return] - except CoalesceError: # no metric metadata present at all - return {} + return _legacy_metric_info(data.attrs, metric_info) def singlets_metric_info( @@ -195,9 +280,10 @@ def singlets_metric_info( be determined - which includes the case of absent metric metadata, since components should not require metadata (see the metadata conventions documentation). """ - return _select_single_metric( - metric_info_by_metric(singletsframe, metric_info, include_inherited), + return _single_metric_info( singletsframe.attrs, + lambda gathered: _current_convention.metric_info(gathered, metric_info, include_inherited), + lambda: _legacy_metric_info(singletsframe.attrs, metric_info), empty=None, ) @@ -210,16 +296,9 @@ def _dataset_default_timezone(attrs: Any) -> str | None: return _string_or_none(extract_from_metadata(attrs, key=_specs.DEFAULT_TIMEZONE)) -def default_timezone_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str | None]: - """The default time zone of each metric, falling back to the one of the dataset metadata - - As specified by the metadata conventions, the default time zone of a metric only falls back - to the one of the dataset metadata, not to its value dimensions or inherited properties. - The default of the returned defaultdict is the one of the dataset metadata, since it applies - to all metrics without their own. Values which are no (non-empty) strings count as missing. - """ - dataset_timezone = _dataset_default_timezone(data.attrs) - gathered = _current_convention.gather(data.attrs) +def _default_timezones( + gathered: _current_convention.GatheredMetadata | None, dataset_timezone: str | None +) -> defaultdict[str, str | None]: own_timezones = ( {} if gathered is None @@ -234,16 +313,31 @@ def default_timezone_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[st ) +def default_timezone_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str | None]: + """The default time zone of each metric, falling back to the one of the dataset metadata + + As specified by the metadata conventions, the default time zone of a metric only falls back + to the one of the dataset metadata, not to its value dimensions or inherited properties. + The default of the returned defaultdict is the one of the dataset metadata, since it applies + to all metrics without their own. Values which are no (non-empty) strings count as missing. + """ + return _default_timezones( + _current_convention.gather(data.attrs), _dataset_default_timezone(data.attrs) + ) + + def single_metric_default_timezone(data: pd.DataFrame | pd.Series) -> str | None: """The default time zone of the single metric of a Series or SingleTSFrame Falls back to the one of the dataset metadata, also if the single metric is not found. """ + dataset_timezone = _dataset_default_timezone(data.attrs) return cast( str | None, - _select_single_metric( - default_timezone_by_metric(data), + _single_metric_info( data.attrs, - empty=_dataset_default_timezone(data.attrs), + lambda gathered: _default_timezones(gathered, dataset_timezone), + dict, # older metadata structures provide no default time zones of the metrics + empty=dataset_timezone, ), ) diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index 9933c58..021a638 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -6,8 +6,15 @@ UNIT: Spec = "unit" MEASUREMENT: Spec = "measurement" NAME: Spec = Coalesce("name", default=None) -DISPLAY_NAME: Spec = Coalesce("display_name", "name", default=None) -SHORT_DISPLAY_NAME: Spec = Coalesce("short_display_name", "display_name", "name", default=None) + +# The fallback chains of the display names. For metadata following the current metadata +# convention, each field is looked up on all levels (value dimension, metric, shared value +# dimensions) before the next field is tried, see _current_convention.value_dimension_info. +# DISPLAY_NAME and SHORT_DISPLAY_NAME are the corresponding specs for older metadata structures. +DISPLAY_NAME_FIELDS = ("display_name", "name") +SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", *DISPLAY_NAME_FIELDS) +DISPLAY_NAME: Spec = Coalesce(*DISPLAY_NAME_FIELDS, default=None) +SHORT_DISPLAY_NAME: Spec = Coalesce(*SHORT_DISPLAY_NAME_FIELDS, default=None) # Key of the dict in a metric object holding the properties the metric inherits from objects # higher up in an asset hierarchy, e.g. the latitude of the plant a sensor belongs to. @@ -17,12 +24,6 @@ # evaluated in by default, e.g. for bucketing days when aggregating. DEFAULT_TIMEZONE = "default_timezone" -# The fallback chains of the display names for metadata following the current metadata -# convention: Each field is looked up on all levels (value dimension, metric, shared value -# dimensions) before the next field is tried, see _current_convention.value_dimension_info. -DISPLAY_NAME_FIELDS = ("display_name", "name") -SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", "display_name", "name") - def by_metric_key_by_val_dimension(metadatum_key: str | Spec) -> Spec: """Provides glom spec that extracts a metadatum by metric by value dimension @@ -145,7 +146,7 @@ def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: def _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - deeper_glom_spec: Spec, add_keys_with_none_values: list[str] | None = None + deeper_glom_spec: Spec, ) -> Spec: """Create dicts with keys from current dict and values from deeper in their value objects @@ -154,9 +155,6 @@ def _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( deeper_glom_spec is the spec to get to the deeper values in each value object. - add_keys_with_none_values allows to add keys even if they do not occur - with a default value of None. - .. doctest:: >>> from hdhelpers.metadata._specs import _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested @@ -195,23 +193,9 @@ def _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( ... ) {'item1': 'kg', 'item2': 'meters', 'item3': 'liters'} """ - if add_keys_with_none_values is None: - add_keys_with_none_values = [] - - start_dict = dict.fromkeys(add_keys_with_none_values, None) - + # T is a stand-in for the target: everything done to T is recorded and executed by glom return ( T.items(), # treat it as list of (key, value) tuples Iter({T[0]: (T[1], deeper_glom_spec)}), Merge(), - lambda x: _update_dict_and_return_it(start_dict.copy(), x), ) - - -# info on T: Basically, think of T as your data’s stunt double. Everything that you do to T will be recorded and executed during the glom() call. -# info to S: On its surface, the glom scope is a dictionary of extra values that can be passed in to the top-level glom call. These values can then be addressed with the S object, which behaves similarly to the T object. -# info on A: Any keyword arguments to the S will have their values evaluated as a spec, with the result being saved to the keyword argument name in the scope. When only the target is being assigned, you can use the A as a shortcut -def _update_dict_and_return_it(start_dict: dict, updated_values_dict: dict) -> dict: - """Update a dict and return it""" - start_dict.update(updated_values_dict) - return start_dict diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index c8c03f7..c5db710 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -17,7 +17,6 @@ from glom import Coalesce, Spec, glom import hdhelpers.metadata._specs as _specs -from hdhelpers.metadata._current_convention import follows_current_convention from hdhelpers.metadata._helpers import ( check_dataframe, check_series, @@ -26,6 +25,7 @@ extract_from_metadata, get_value_dimension_info, metric_info_by_metric, + series_value_info, singlets_metric_info, singlets_value_dimension_info, single_metric_default_timezone, @@ -391,16 +391,24 @@ def get_series_info( True """ - # Since the fallback behaviour for this value dimension is to fall back to the metric - # metadata, we can reuse the code that extracts value_dimension metadata for - # this value dimension. check_series(series) - from_new_convention = _series_value( - get_value_dimension_info(series, value_dim_info, include_inherited), series - ) + return _series_info(series, (value_dim_info,), value_dim_info, include_inherited) + - if from_new_convention is not None: - return from_new_convention +def _series_info( + series: pd.Series, fields: tuple[str | Spec, ...], legacy_spec: Spec, include_inherited: bool +) -> Any: + """Info of a Series, falling back to further fields + + Since a Series has only one value dimension named "value", it falls back to the metric + metadata. For metadata following the current metadata convention, each field is looked up + on the value dimension, the metric and "value_dimensions_shared" before the next field is + tried, see value_dimension_info_with_fallbacks. Older metadata structures are handled via + legacy_spec, keeping their established behaviour. + """ + from_metrics = series_value_info(series, fields, legacy_spec, include_inherited) + if from_metrics is not None: + return from_metrics # compatibility with some older format return glom( @@ -409,13 +417,13 @@ def get_series_info( spec_not_none( ( "single_metric_metadata.structured_metadata.value_dimensions.value", - value_dim_info, + legacy_spec, ) ), spec_not_none( ( "single_metric_metadata.structured_metadata.metric", - value_dim_info, + legacy_spec, ) ), default=None, @@ -423,41 +431,16 @@ def get_series_info( ) -def _series_value(info_by_metric: dict, series: pd.Series) -> Any: - """Info of the value dimension "value" of the metric of a Series - - The metric is identified via "single_metric" in "dataset_metadata". If that is missing, the - metric "series" as used by older metadata structures is used or else the only metric - present, since a Series cannot be ambiguous in this respect. An identified metric which is - not present yields None, since the only metric present is another one then. - """ - series_metric_key = extract_from_metadata(series.attrs, key="single_metric") - if series_metric_key is not None: - return info_by_metric.get(series_metric_key, {}).get("value") - if "series" in info_by_metric: - return info_by_metric["series"].get("value") - if len(info_by_metric) == 1: - return next(iter(info_by_metric.values())).get("value") - return None - - def _series_info_with_fallbacks( - series: pd.Series, fields: tuple[str, ...], include_inherited: bool + series: pd.Series, fields: tuple[str | Spec, ...], include_inherited: bool ) -> Any: - """Info of a Series, falling back to further fields + """Info of a Series, falling back to further fields, which may be provided as null - For metadata following the current metadata convention, each field is looked up on the - value dimension and the metric before the next field is tried, see - value_dimension_info_with_fallbacks. Older metadata structures keep their established - behaviour of using the first of the fields present on the first level providing any. + Older metadata structures keep their established behaviour of using the first of the + fields present on the first level providing any. """ legacy_spec = Coalesce(*(spec_not_none(field) for field in fields)) - if follows_current_convention(series.attrs): - return _series_value( - value_dimension_info_with_fallbacks(series, fields, legacy_spec, include_inherited), - series, - ) - return get_series_info(series, legacy_spec, include_inherited=include_inherited) + return _series_info(series, fields, legacy_spec, include_inherited) def get_series_unit(series: pd.Series, *, include_inherited: bool = False) -> str | None: @@ -505,7 +488,7 @@ def get_series_unit(series: pd.Series, *, include_inherited: bool = False) -> st check_series(series) return cast( str | None, - get_series_info(series, spec_not_none("unit"), include_inherited=include_inherited), + _series_info_with_fallbacks(series, (_specs.UNIT,), include_inherited), ) @@ -556,7 +539,7 @@ def get_series_name(series: pd.Series, *, include_inherited: bool = False) -> st check_series(series) return cast( str | None, - get_series_info(series, spec_not_none("name"), include_inherited=include_inherited), + _series_info_with_fallbacks(series, ("name",), include_inherited), ) @@ -588,9 +571,7 @@ def get_series_display_name(series: pd.Series, *, include_inherited: bool = Fals check_series(series) return cast( str | None, - _series_info_with_fallbacks( - series, ("display_name", "name", "short_display_name"), include_inherited - ), + _series_info_with_fallbacks(series, _specs.DISPLAY_NAME_FIELDS, include_inherited), ) @@ -624,9 +605,7 @@ def get_series_short_display_name( check_series(series) return cast( str | None, - _series_info_with_fallbacks( - series, ("short_display_name", "display_name", "name"), include_inherited - ), + _series_info_with_fallbacks(series, _specs.SHORT_DISPLAY_NAME_FIELDS, include_inherited), ) diff --git a/tests/metadata/test_metadata_default_timezone.py b/tests/metadata/test_metadata_default_timezone.py index 336349e..dc3504c 100644 --- a/tests/metadata/test_metadata_default_timezone.py +++ b/tests/metadata/test_metadata_default_timezone.py @@ -113,6 +113,14 @@ def test_default_timezone_does_not_fall_back_to_other_levels(): }, None, ), + # a single_metric which is no string counts as missing + ( + { + "dataset_metadata": {"single_metric": ["a"]}, + "metrics": [{"id": "a", "default_timezone": "UTC"}], + }, + "UTC", + ), # without metric metadata the dataset metadata still applies ({"dataset_metadata": {"default_timezone": "Europe/Berlin"}}, "Europe/Berlin"), ( diff --git a/tests/metadata/test_metadata_fallbacks.py b/tests/metadata/test_metadata_fallbacks.py index 7e98d43..3455339 100644 --- a/tests/metadata/test_metadata_fallbacks.py +++ b/tests/metadata/test_metadata_fallbacks.py @@ -15,7 +15,9 @@ get_measurements, get_metric_info, get_names, + get_series_default_timezone, get_series_display_name, + get_series_info, get_series_measurement, get_series_name, get_series_short_display_name, @@ -221,6 +223,54 @@ def test_single_metric_is_not_replaced_by_another_only_metric(): assert get_singlets_metric_info(frame(attrs), "name") is None +@pytest.mark.parametrize("single_metric", [None, 1, [SIGNAL_ID]]) +def test_single_metric_which_is_no_string_counts_as_missing(single_metric): + attrs = deepcopy(platform_series_attrs) + attrs["dataset_metadata"]["single_metric"] = single_metric + + assert get_series_unit(series(attrs)) == "m" + assert get_singlets_units(frame(attrs))["value"] == "m" + assert get_singlets_metric_info(frame(attrs), "name") == "Water-lvl-north" + + +def test_series_with_metadata_of_several_metrics_has_no_metric(): + """E.g. a Series taken from a MultiTSFrame, whose metadata it carries + + A metric named "series" is only the metric of a Series in older metadata structures. + """ + attrs = { + "dataset_metadata": {"default_timezone": "UTC"}, + "metrics": [ + metric(id="series", name="Series", default_timezone="Asia/Tokyo"), + metric(id="other", name="Other"), + ], + } + + assert get_series_name(series(attrs)) is None + assert get_series_default_timezone(series(attrs)) == "UTC" + + +def test_only_the_single_metric_is_resolved(): + """The spec is not applied to other metrics, so failing on their null names does not matter""" + attrs = deepcopy(platform_series_attrs) + attrs["metrics"].append(metric(id="other")) + + assert get_series_info(series(attrs), ("name", str.upper)) == "WATER-LVL-NORTH" + assert get_singlets_metric_info(frame(attrs), ("name", str.upper)) == "WATER-LVL-NORTH" + + +def test_series_display_name_does_not_fall_back_to_short_display_name(): + """Like the display names of MultiTSFrames and SingleTSFrames""" + attrs = deepcopy(platform_series_attrs) + attrs["metrics"][0]["name"] = None + attrs["metrics"][0]["value_dimensions"][0]["name"] = None + attrs["metrics"][0]["short_display_name"] = "north" + + assert get_series_display_name(series(attrs)) is None + assert get_singlets_display_names(frame(attrs))["value"] is None + assert get_series_short_display_name(series(attrs)) == "north" + + @pytest.mark.parametrize( "attrs", [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}], From dd0cd10956542ae9059e75634052812e43fa76bf Mon Sep 17 00:00:00 2001 From: stewit <> Date: Thu, 22 Oct 2026 19:52:03 +0200 Subject: [PATCH 11/15] add recording --- CHANGELOG.md | 3 + src/hdhelpers/metadata/__init__.py | 18 ++ src/hdhelpers/metadata/_recording.py | 305 ++++++++++++++++++++ src/hdhelpers/metadata/helpers.py | 318 ++++++++++++++++++++ tests/metadata/test_metadata_recording.py | 337 ++++++++++++++++++++++ 5 files changed, 981 insertions(+) create mode 100644 src/hdhelpers/metadata/_recording.py create mode 100644 tests/metadata/test_metadata_recording.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7669966..690a341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ - entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error - new functions get_default_timezones, get_series_default_timezone and get_singlets_default_timezone provide the default time zone of the metrics ("default_timezone" of the current metadata convention), falling back to the one of the dataset metadata. It is meant for computations, e.g. bucketing days when aggregating, not for displaying timestamps, which is what the time zone of the plot target settings is for. New section "Which time zone to compute in?" in the documentation - documentation: the information requested from get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info is interpreted as a glom spec. New section "How to query nested information?" with examples for dotted paths into sub dictionaries, other glom specs and keys containing dots +- new class Recording and functions get_recordings, get_series_recording and get_singlets_recording provide how the values of the metrics are recorded: "recording_mode", "data_frequency" with "data_frequency_offset" and "data_frequency_timezone", and "recording_aggregation_method" of the current metadata convention. The frequency falls back to "ref_data_frequency", "ref_data_frequency_offset" and "ref_data_frequency_timezone" of the dataset metadata as a whole, i.e. offset and time zone only apply together with the frequency of the same level. A missing offset means 0, a missing time zone UTC +- new functions recording_interval_starts and recording_grid provide the starts of the recording intervals of a Recording: multiples of the frequency since 1970-01-01T00:00:00Z plus the offset, or with a time zone, intervals restarting at each local midnight (the last one of a day cut at the next midnight, e.g. on days with a daylight saving time change), local calendar days or local calendar months +- new functions get_default_aggregation_methods, get_series_default_aggregation_method and get_singlets_default_aggregation_method provide the default aggregation method of the metrics ("default_aggregation_method"), normalizing "AVG" to "MEAN". Without one, it is "LAST" for metrics whose value dimension "value" has the value data type "boolean" or "string", otherwise "MEAN" ## [0.0.9] - 2026-09-10 - singletsframe support diff --git a/src/hdhelpers/metadata/__init__.py b/src/hdhelpers/metadata/__init__.py index cc48e5d..0a1d1a9 100644 --- a/src/hdhelpers/metadata/__init__.py +++ b/src/hdhelpers/metadata/__init__.py @@ -3,6 +3,15 @@ extract the requested information.""" from hdhelpers.metadata.helpers import ( + Recording, + get_default_aggregation_methods, + get_recordings, + get_series_default_aggregation_method, + get_series_recording, + get_singlets_default_aggregation_method, + get_singlets_recording, + recording_grid, + recording_interval_starts, get_default_timezones, get_display_names, get_measurements, @@ -52,4 +61,13 @@ "get_default_timezones", "get_series_default_timezone", "get_singlets_default_timezone", + "Recording", + "get_recordings", + "get_series_recording", + "get_singlets_recording", + "recording_interval_starts", + "recording_grid", + "get_default_aggregation_methods", + "get_series_default_aggregation_method", + "get_singlets_default_aggregation_method", ] diff --git a/src/hdhelpers/metadata/_recording.py b/src/hdhelpers/metadata/_recording.py new file mode 100644 index 0000000..4064196 --- /dev/null +++ b/src/hdhelpers/metadata/_recording.py @@ -0,0 +1,305 @@ +"""How the values of the metrics are recorded and how they should be aggregated by default + +The frequency of the recorded values of a metric falls back as a whole: If the metric has a +"data_frequency", it applies together with the metric's "data_frequency_offset" and +"data_frequency_timezone". Otherwise "ref_data_frequency" of the dataset metadata applies +together with "ref_data_frequency_offset" and "ref_data_frequency_timezone". A missing offset +is 0. + +The recording intervals start at the multiples of the frequency plus the offset. Without time +zone these are the multiples since 1970-01-01T00:00:00Z. With time zone and a frequency shorter +than a day, the intervals restart at each local midnight (plus offset) of the time zone, counted +in elapsed time, and the last interval of a day ends at the next local midnight. With time zone +and a frequency of a day or longer, the intervals follow the calendar of the time zone, e.g. +local days of 23, 24 or 25 hours. +""" + +import math +from collections import defaultdict +from dataclasses import dataclass +from typing import Any, TypeIs, cast + +import pandas as pd + +import hdhelpers.metadata._current_convention as _current_convention +from hdhelpers.metadata._helpers import _select_single_metric, _string_or_none + +_FREQUENCY_FIELDS = ("data_frequency", "data_frequency_offset", "data_frequency_timezone") +_DATASET_FREQUENCY_FIELDS = ( + "ref_data_frequency", + "ref_data_frequency_offset", + "ref_data_frequency_timezone", +) + +# default aggregation method of metrics without one by the data type of their values +_DEFAULT_AGGREGATION_METHOD_BY_VALUE_DATA_TYPE = {"boolean": "LAST", "string": "LAST"} +_DEFAULT_AGGREGATION_METHOD = "MEAN" +_AGGREGATION_METHOD_ALIASES = {"AVG": "MEAN"} + +_DAY = pd.Timedelta(days=1) + + +@dataclass(frozen=True) +class Recording: + """How the values of a metric are recorded + + The fields are taken from the metadata as given. data_frequency_offset and + data_frequency_timezone belong to data_frequency: They are None if no data_frequency + applies. A data_frequency_offset of None means 0. + """ + + recording_mode: str | None = None + data_frequency: str | None = None + data_frequency_offset: str | None = None + data_frequency_timezone: str | None = None + recording_aggregation_method: str | None = None + + +_Frequency = tuple[str | None, str | None, str | None] +_NO_FREQUENCY: _Frequency = (None, None, None) + + +def _frequency(obj: Any, fields: tuple[str, str, str]) -> _Frequency: + """Frequency, offset and time zone of obj, all None if it has no frequency + + Values which are no (non-empty) strings count as missing. + """ + if not isinstance(obj, dict): + return _NO_FREQUENCY + frequency, offset, timezone = (_string_or_none(obj.get(field)) for field in fields) + if frequency is None: + return _NO_FREQUENCY + return frequency, offset, timezone + + +def _dataset_frequency(attrs: Any) -> _Frequency: + dataset_metadata = attrs.get("dataset_metadata") if isinstance(attrs, dict) else None + return _frequency(dataset_metadata, _DATASET_FREQUENCY_FIELDS) + + +def _recording(metric: dict, dataset_frequency: _Frequency) -> Recording: + own_frequency = _frequency(metric, _FREQUENCY_FIELDS) + frequency, offset, timezone = ( + own_frequency if own_frequency[0] is not None else dataset_frequency + ) + return Recording( + recording_mode=_string_or_none(metric.get("recording_mode")), + data_frequency=frequency, + data_frequency_offset=offset, + data_frequency_timezone=timezone, + recording_aggregation_method=_string_or_none(metric.get("recording_aggregation_method")), + ) + + +def recording_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, Recording]: + """The recording of each metric, falling back to the frequency of the dataset metadata + + The default of the returned defaultdict is the recording of a metric without metadata, i.e. + with the frequency of the dataset metadata. + """ + dataset_frequency = _dataset_frequency(data.attrs) + gathered = _current_convention.gather(data.attrs) + metrics = ( + {} + if gathered is None + else { + metric: gathered_metric.metric for metric, gathered_metric in gathered.metrics.items() + } + ) + return defaultdict( + lambda: Recording(None, *dataset_frequency, None), + {metric: _recording(obj, dataset_frequency) for metric, obj in metrics.items()}, + ) + + +def single_metric_recording(data: pd.DataFrame | pd.Series) -> Recording: + """The recording of the single metric of a Series or SingleTSFrame + + Falls back to the frequency of the dataset metadata, also if the single metric is not found. + """ + return cast( + Recording, + _select_single_metric( + recording_by_metric(data), + data.attrs, + empty=Recording(None, *_dataset_frequency(data.attrs), None), + ), + ) + + +def _default_aggregation_method(method: Any, value_data_type: Any) -> str: + own_method = _string_or_none(method) + if own_method is not None: + return _AGGREGATION_METHOD_ALIASES.get(own_method, own_method) + return _DEFAULT_AGGREGATION_METHOD_BY_VALUE_DATA_TYPE.get( + value_data_type, _DEFAULT_AGGREGATION_METHOD + ) + + +def default_aggregation_method_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str]: + """The default aggregation method of each metric + + The method of the metric, with "AVG" normalized to its alias "MEAN". If the metric does not + provide one, "LAST" for values of the data type "boolean" or "string" (of the value + dimension "value", see value_dimension_info) and "MEAN" otherwise, which is also the default + of the returned defaultdict. + """ + gathered = _current_convention.gather(data.attrs) + if gathered is None: + return defaultdict(lambda: _DEFAULT_AGGREGATION_METHOD) + methods = _current_convention.own_metric_info(gathered, "default_aggregation_method") + value_data_types = _current_convention.value_dimension_info( + gathered, ["value_data_type"], include_inherited=False + ) + return defaultdict( + lambda: _DEFAULT_AGGREGATION_METHOD, + { + metric: _default_aggregation_method(method, value_data_types[metric]["value"]) + for metric, method in methods.items() + }, + ) + + +def single_metric_default_aggregation_method(data: pd.DataFrame | pd.Series) -> str: + """The default aggregation method of the single metric of a Series or SingleTSFrame""" + return cast( + str, + _select_single_metric( + default_aggregation_method_by_metric(data), + data.attrs, + empty=_DEFAULT_AGGREGATION_METHOD, + ), + ) + + +def _grid_parameters( + recording: Recording, +) -> tuple[pd.offsets.BaseOffset, pd.Timedelta, str | None]: + """Frequency as Pandas offset, offset as Timedelta and time zone of a recording""" + if recording.data_frequency is None: + raise ValueError("The recording has no data_frequency, which defines its intervals.") + try: + frequency = pd.tseries.frequencies.to_offset(recording.data_frequency) + except ValueError: + # e.g. "P1D" or "1 days" + frequency = pd.tseries.frequencies.to_offset( + pd.to_timedelta(recording.data_frequency) # type: ignore[call-overload] + ) + offset = pd.to_timedelta(recording.data_frequency_offset or "0s") + return frequency, offset, recording.data_frequency_timezone + + +def _is_shorter_than_a_day(frequency: pd.offsets.BaseOffset) -> TypeIs[pd.offsets.Tick]: + return isinstance(frequency, pd.offsets.Tick) and pd.Timedelta(frequency) < _DAY + + +def _utc_series(timestamps: pd.DatetimeIndex | pd.Series) -> pd.Series: + """Timestamps as Series in UTC, naive timestamps are considered to be UTC""" + series = pd.Series(timestamps) if isinstance(timestamps, pd.DatetimeIndex) else timestamps + if series.dt.tz is None: + return series.dt.tz_localize("UTC") + return series.dt.tz_convert("UTC") + + +def _localize_midnights(naive_midnights: pd.Series, timezone: str) -> pd.Series: + """Localize local midnights, shifting nonexistent ones forward and preferring DST""" + return naive_midnights.dt.tz_localize( + timezone, ambiguous=[True] * len(naive_midnights), nonexistent="shift_forward" + ) + + +def _calendar_interval_starts( + utc: pd.Series, frequency: pd.offsets.BaseOffset, offset: pd.Timedelta, timezone: str +) -> pd.Series: + """Interval starts of a frequency of a day or longer in the calendar of the time zone""" + local_days = (utc - offset).dt.tz_convert(timezone).dt.tz_localize(None).dt.normalize() + if isinstance(frequency, pd.offsets.Day | pd.offsets.Tick): + step = ( + frequency.n * _DAY if isinstance(frequency, pd.offsets.Day) else pd.Timedelta(frequency) + ) + if step % _DAY != pd.Timedelta(0): + raise ValueError( + f'A frequency of "{frequency.freqstr}" cannot follow the calendar of a time zone,' + " only multiples of a day can." + ) + # multiples of whole days since 1970-01-01 in local time + epoch = pd.Timestamp("1970-01-01") + starts = epoch + ((local_days - epoch) // step) * step + else: + # e.g. the first day of the month for "MS" + unique_starts = {day: frequency.rollback(day) for day in local_days.dropna().unique()} + starts = local_days.map(unique_starts).astype(local_days.dtype) + return _localize_midnights(starts, timezone) + offset + + +def recording_interval_starts[T: (pd.DatetimeIndex, pd.Series)]( + timestamps: T, recording: Recording +) -> T: + """The start of the recording interval of each timestamp, in UTC""" + frequency, offset, timezone = _grid_parameters(recording) + utc = _utc_series(timestamps) + + if timezone is None and isinstance(frequency, pd.offsets.Tick): + epoch = pd.Timestamp(0, tz="UTC") + duration = pd.Timedelta(frequency) + starts = epoch + offset + ((utc - epoch - offset) // duration) * duration + elif timezone is not None and _is_shorter_than_a_day(frequency): + duration = pd.Timedelta(frequency) + # the intervals of a day start at its local midnight plus offset + local_midnights = (utc - offset).dt.tz_convert(timezone).dt.normalize() + day_starts = local_midnights.dt.tz_convert("UTC") + offset + starts = day_starts + ((utc - day_starts) // duration) * duration + else: + starts = _calendar_interval_starts(utc, frequency, offset, timezone or "UTC").dt.tz_convert( + "UTC" + ) + + if isinstance(timestamps, pd.DatetimeIndex): + return pd.DatetimeIndex(starts.to_numpy(), name=timestamps.name) + return starts.set_axis(timestamps.index) + + +def recording_grid( + start: pd.Timestamp, end: pd.Timestamp, recording: Recording +) -> pd.DatetimeIndex: + """The recording interval starts from start to end, both inclusive, in UTC""" + frequency, offset, timezone = _grid_parameters(recording) + start = _utc_series(pd.DatetimeIndex([start])).iloc[0] + end = _utc_series(pd.DatetimeIndex([end])).iloc[0] + if start > end: + return pd.DatetimeIndex([], tz="UTC") + first = recording_interval_starts(pd.DatetimeIndex([start]), recording)[0] + + if timezone is None and isinstance(frequency, pd.offsets.Tick): + grid = pd.date_range(first, end, freq=pd.Timedelta(frequency)) + elif timezone is not None and _is_shorter_than_a_day(frequency): + duration = pd.Timedelta(frequency) + local_midnights = pd.date_range( + (first - offset).tz_convert(timezone).normalize(), + (end - offset).tz_convert(timezone).normalize() + _DAY, + freq="D", + ) + day_starts = local_midnights.tz_convert("UTC") + offset + grid = pd.DatetimeIndex( + [ + day_start + index * duration + for day_start, next_day_start in zip(day_starts[:-1], day_starts[1:], strict=True) + for index in range(math.ceil((next_day_start - day_start) / duration)) + ], + tz="UTC", + ) + else: + # step through the calendar of the time zone via the interval starts of the days + days = pd.date_range( + (first - offset).tz_convert(timezone or "UTC").normalize().tz_localize(None), + (end - offset).tz_convert(timezone or "UTC").normalize().tz_localize(None), + freq="D", + ) + candidates = ( + _localize_midnights(pd.Series(days), timezone or "UTC").dt.tz_convert("UTC") + offset + ) + grid = pd.DatetimeIndex( + recording_interval_starts(pd.DatetimeIndex(candidates), recording).unique() + ) + + return grid[(grid >= start) & (grid <= end)] diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index c5db710..d494c8a 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -17,6 +17,17 @@ from glom import Coalesce, Spec, glom import hdhelpers.metadata._specs as _specs +from hdhelpers.metadata._recording import ( + Recording, + default_aggregation_method_by_metric, + recording_by_metric, + single_metric_default_aggregation_method, + single_metric_recording, +) +from hdhelpers.metadata._recording import recording_grid as _recording_grid +from hdhelpers.metadata._recording import ( + recording_interval_starts as _recording_interval_starts, +) from hdhelpers.metadata._helpers import ( check_dataframe, check_series, @@ -1147,3 +1158,310 @@ def get_singlets_default_timezone(singletsframe: pd.DataFrame) -> str | None: """ check_dataframe(singletsframe) return single_metric_default_timezone(singletsframe) + + +def get_recordings(multitsframe: pd.DataFrame) -> defaultdict[str, Recording]: + """Gets how the values of each metric of a MultiTSFrame are recorded + + The recording consists of the "recording_mode" of the metric (e.g. "periodic" or + "on_change"), the frequency of its values with offset and time zone and its + "recording_aggregation_method", see "Recording of the values" in the metadata conventions. + + The frequency falls back as a whole: If the metric has a "data_frequency", it applies + together with the metric's "data_frequency_offset" and "data_frequency_timezone". Otherwise + "ref_data_frequency" of the dataset metadata applies together with + "ref_data_frequency_offset" and "ref_data_frequency_timezone". The values are provided as + given in the metadata, an offset of None means 0. Use :func:`recording_interval_starts` and + :func:`recording_grid` for the timestamps of the recording intervals. + + Args: + multitsframe (pd.DataFrame): MTSF with metadata following the convention. + + Returns: + defaultdict[str, Recording]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are their recordings. Defaults to a recording with the frequency of the dataset metadata, also for metrics without metadata. + + Raises: + TypeError: If `multitsframe` is not a DataFrame. + + Code example: + + .. doctest:: metadata.get_recordings + + >>> multitsframe.attrs = { + ... "dataset_metadata": {"metric_key": "id", "ref_data_frequency": "1h"}, + ... "metrics": [ + ... { + ... "id": "average_level", + ... "recording_mode": "periodic", + ... "data_frequency": "15min", + ... "recording_aggregation_method": "MEAN", + ... }, + ... {"id": "pump_state", "recording_mode": "on_change"}, + ... ], + ... } + >>> recordings = get_recordings(multitsframe) + >>> recordings["average_level"].data_frequency + '15min' + >>> recordings["pump_state"].recording_mode + 'on_change' + >>> recordings["pump_state"].data_frequency + '1h' + """ + check_dataframe(multitsframe) + return recording_by_metric(multitsframe) + + +def get_series_recording(series: pd.Series) -> Recording: + """Gets how the values of a Series are recorded + + See :func:`get_recordings` for the recording and how its frequency falls back to the one of + the dataset metadata. + + Args: + series (pd.Series): Series with metadata following the convention. + + Returns: + Recording: The recording of the metric, with the frequency of the dataset metadata if the metric does not provide one. + + Raises: + TypeError: If `series` is not a Series. + + Code example: + + .. doctest:: metadata.get_series_recording + + >>> series.attrs = { + ... "dataset_metadata": {"single_metric": "water.north.level", "metric_key": "external_id"}, + ... "metrics": [ + ... { + ... "external_id": "water.north.level", + ... "recording_mode": "periodic", + ... "data_frequency": "2h", + ... "data_frequency_timezone": "Europe/Berlin", + ... } + ... ], + ... } + >>> recording = get_series_recording(series) + >>> recording.data_frequency, recording.data_frequency_offset, recording.data_frequency_timezone + ('2h', None, 'Europe/Berlin') + """ + check_series(series) + return single_metric_recording(series) + + +def get_singlets_recording(singletsframe: pd.DataFrame) -> Recording: + """Gets how the values of the single metric of a SingleTSFrame are recorded + + See :func:`get_recordings` for the recording and how its frequency falls back to the one of + the dataset metadata. + + Args: + singletsframe (pd.DataFrame): STSF with metadata following the convention. + + Returns: + Recording: The recording of the metric, with the frequency of the dataset metadata if the metric does not provide one. + + Raises: + TypeError: If `singletsframe` is not a DataFrame. + + Code example: + + .. doctest:: metadata.get_singlets_recording + + >>> dataframe.attrs = { + ... "dataset_metadata": { + ... "single_metric": "abc.temp", + ... "ref_data_frequency": "5min", + ... "ref_data_frequency_offset": "1min", + ... }, + ... "metrics": [{"id": "abc.temp", "recording_mode": "periodic"}], + ... } + >>> recording = get_singlets_recording(dataframe) + >>> recording.recording_mode, recording.data_frequency, recording.data_frequency_offset + ('periodic', '5min', '1min') + """ + check_dataframe(singletsframe) + return single_metric_recording(singletsframe) + + +def get_default_aggregation_methods(multitsframe: pd.DataFrame) -> defaultdict[str, str]: + """Gets the method each metric of a MultiTSFrame should be aggregated with by default + + E.g. for plotting or reporting it in coarser intervals. It is the "default_aggregation_method" + of the metric, with "AVG" normalized to its alias "MEAN". If the metric does not provide one, + it is "LAST" for values of the data type "boolean" or "string" (the "value_data_type" of the + value dimension "value"), i.e. the state holding at the end of the interval, and "MEAN" + otherwise. As specified by the metadata conventions, it is not taken from inherited + properties. + + Args: + multitsframe (pd.DataFrame): MTSF with metadata following the convention. + + Returns: + defaultdict[str, str]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are their default aggregation methods, e.g. "MEAN", "SUM" or "LAST". Defaults to "MEAN" for metrics without metadata. + + Raises: + TypeError: If `multitsframe` is not a DataFrame. + + Code example: + + .. doctest:: metadata.get_default_aggregation_methods + + >>> multitsframe.attrs = { + ... "dataset_metadata": {"metric_key": "id"}, + ... "metrics": [ + ... {"id": "level", "default_aggregation_method": "AVG"}, + ... {"id": "rainfall", "default_aggregation_method": "SUM"}, + ... { + ... "id": "pump_running", + ... "value_dimensions": [{"column": "value", "value_data_type": "boolean"}], + ... }, + ... ], + ... } + >>> methods = get_default_aggregation_methods(multitsframe) + >>> methods["level"], methods["rainfall"], methods["pump_running"] + ('MEAN', 'SUM', 'LAST') + >>> methods["metric_without_metadata"] + 'MEAN' + """ + check_dataframe(multitsframe) + return default_aggregation_method_by_metric(multitsframe) + + +def get_series_default_aggregation_method(series: pd.Series) -> str: + """Gets the method a Series should be aggregated with by default + + See :func:`get_default_aggregation_methods` for the method and its default. + + Args: + series (pd.Series): Series with metadata following the convention. + + Returns: + str: The default aggregation method, e.g. "MEAN", "SUM" or "LAST". + + Raises: + TypeError: If `series` is not a Series. + + Code example: + + .. doctest:: metadata.get_series_default_aggregation_method + + >>> series.attrs = { + ... "dataset_metadata": {"single_metric": "rainfall"}, + ... "metrics": [{"id": "rainfall", "default_aggregation_method": "SUM"}], + ... } + >>> get_series_default_aggregation_method(series) + 'SUM' + >>> series.attrs = {} + >>> get_series_default_aggregation_method(series) + 'MEAN' + """ + check_series(series) + return single_metric_default_aggregation_method(series) + + +def get_singlets_default_aggregation_method(singletsframe: pd.DataFrame) -> str: + """Gets the method the single metric of a SingleTSFrame should be aggregated with by default + + See :func:`get_default_aggregation_methods` for the method and its default. + + Args: + singletsframe (pd.DataFrame): STSF with metadata following the convention. + + Returns: + str: The default aggregation method, e.g. "MEAN", "SUM" or "LAST". + + Raises: + TypeError: If `singletsframe` is not a DataFrame. + + Code example: + + .. doctest:: metadata.get_singlets_default_aggregation_method + + >>> dataframe.attrs = { + ... "dataset_metadata": {"single_metric": "valve"}, + ... "metrics": [ + ... { + ... "id": "valve", + ... "value_dimensions": [{"column": "value", "value_data_type": "string"}], + ... } + ... ], + ... } + >>> get_singlets_default_aggregation_method(dataframe) + 'LAST' + """ + check_dataframe(singletsframe) + return single_metric_default_aggregation_method(singletsframe) + + +def recording_interval_starts[T: (pd.DatetimeIndex, pd.Series)]( + timestamps: T, recording: Recording +) -> T: + """Gets the start of the recording interval of each timestamp + + The recording intervals start at the multiples of the frequency of the recording plus its + offset (0 if None): + + * Without time zone, these are the multiples since 1970-01-01T00:00:00Z. + * With time zone and a frequency shorter than a day, the intervals restart at each local + midnight (plus offset) of the time zone, counted in elapsed time. The last interval of a + day ends at the next local midnight, so it is shorter on days with a daylight saving time + change or if the frequency does not divide a day. This is how e.g. the hetida platform + compresses values in windows. + * With time zone and a frequency of a day or longer, e.g. "1D" or "1MS", the intervals follow + the calendar of the time zone, e.g. local days of 23, 24 or 25 hours. + + Args: + timestamps (pd.DatetimeIndex | pd.Series): The timestamps, naive ones are considered to be UTC. + recording (Recording): The recording, e.g. from :func:`get_series_recording`. + + Returns: + pd.DatetimeIndex | pd.Series: The start of the recording interval of each timestamp in UTC, as DatetimeIndex or Series (with the index of `timestamps`) like `timestamps`. + + Raises: + ValueError: If the recording has no data_frequency or its frequency, offset or time zone cannot be interpreted. + + Code example: + + .. doctest:: metadata.recording_interval_starts + + >>> recording = Recording(data_frequency="2h", data_frequency_timezone="Europe/Berlin") + >>> # on 2026-03-29 the clocks in Berlin are put forward at 2:00 local time (1:00 UTC) + >>> starts = recording_interval_starts( + ... pd.DatetimeIndex(["2026-03-29T00:30:00Z", "2026-03-29T01:30:00Z"]), recording + ... ) + >>> [start.isoformat() for start in starts.tz_convert("Europe/Berlin")] + ['2026-03-29T00:00:00+01:00', '2026-03-29T03:00:00+02:00'] + """ + return _recording_interval_starts(timestamps, recording) + + +def recording_grid( + start: pd.Timestamp | str, end: pd.Timestamp | str, recording: Recording +) -> pd.DatetimeIndex: + """Gets the starts of the recording intervals from start to end, both inclusive + + For a periodic recording, these are the timestamps at which values are expected. See + :func:`recording_interval_starts` for how the intervals are determined. + + Args: + start (pd.Timestamp | str): Start, naive timestamps are considered to be UTC. + end (pd.Timestamp | str): End, naive timestamps are considered to be UTC. + recording (Recording): The recording, e.g. from :func:`get_series_recording`. + + Returns: + pd.DatetimeIndex: The interval starts in UTC. + + Raises: + ValueError: If the recording has no data_frequency or its frequency, offset or time zone cannot be interpreted. + + Code example: + + .. doctest:: metadata.recording_grid + + >>> recording = Recording(data_frequency="15min", data_frequency_offset="5min") + >>> grid = recording_grid("2026-01-01T00:00:00Z", "2026-01-01T00:40:00Z", recording) + >>> [timestamp.isoformat() for timestamp in grid] + ['2026-01-01T00:05:00+00:00', '2026-01-01T00:20:00+00:00', '2026-01-01T00:35:00+00:00'] + """ + return _recording_grid(pd.Timestamp(start), pd.Timestamp(end), recording) diff --git a/tests/metadata/test_metadata_recording.py b/tests/metadata/test_metadata_recording.py new file mode 100644 index 0000000..628b7b5 --- /dev/null +++ b/tests/metadata/test_metadata_recording.py @@ -0,0 +1,337 @@ +"""Recording of the metrics, their default aggregation methods and the recording intervals""" + +import pandas as pd +import pytest + +from hdhelpers.metadata import ( + Recording, + get_default_aggregation_methods, + get_recordings, + get_series_default_aggregation_method, + get_series_recording, + get_singlets_default_aggregation_method, + get_singlets_recording, + recording_grid, + recording_interval_starts, +) + + +def frame(attrs): + df = pd.DataFrame() + df.attrs = attrs + return df + + +def series(attrs): + ser = pd.Series(dtype=float) + ser.attrs = attrs + return ser + + +def value_data_type(data_type): + return [{"column": "value", "unit": None, "measurement": None, "value_data_type": data_type}] + + +multitsframe_attrs = { + "dataset_metadata": { + "metric_key": "external_id", + "ref_data_frequency": "1h", + "ref_data_frequency_offset": "30min", + "ref_data_frequency_timezone": "Europe/Berlin", + }, + "metrics": [ + { + "external_id": "own_frequency", + "recording_mode": "periodic", + "data_frequency": "15min", + "recording_aggregation_method": "MEAN", + }, + # the offset and time zone of the metric only apply together with its frequency + { + "external_id": "without_frequency", + "recording_mode": "on_change", + "data_frequency_offset": "5min", + "data_frequency_timezone": "UTC", + }, + {"external_id": "null_frequency", "data_frequency": None}, + {"external_id": "no_string", "data_frequency": 15, "recording_mode": 1}, + ], +} + +dataset_frequency = { + "data_frequency": "1h", + "data_frequency_offset": "30min", + "data_frequency_timezone": "Europe/Berlin", +} + + +def test_recordings_fall_back_to_the_frequency_of_the_dataset_as_a_whole(): + recordings = get_recordings(frame(multitsframe_attrs)) + + assert recordings == { + "own_frequency": Recording( + recording_mode="periodic", + data_frequency="15min", + recording_aggregation_method="MEAN", + ), + "without_frequency": Recording(recording_mode="on_change", **dataset_frequency), + "null_frequency": Recording(**dataset_frequency), + "no_string": Recording(**dataset_frequency), + } + assert recordings["metric without metadata"] == Recording(**dataset_frequency) + + +@pytest.mark.parametrize( + "attrs", + [ + {}, + {"metrics": "no list"}, + # an older metadata structure, whose metrics are not considered + {"dataset_metadata": {}, "metrics": {"a": {"data_frequency": "1h"}}}, + ], +) +def test_recordings_without_metric_metadata(attrs): + recordings = get_recordings(frame(attrs)) + + assert recordings == {} + assert recordings["a"] == Recording() + + +def test_series_recording(): + attrs = { + "dataset_metadata": {"single_metric": "b", "ref_data_frequency": "1D"}, + "metrics": [{"id": "b", "recording_mode": "periodic", "data_frequency_offset": "1h"}], + } + + assert get_series_recording(series(attrs)) == Recording( + recording_mode="periodic", data_frequency="1D" + ) + + +def test_singlets_recording_falls_back_to_the_dataset_if_the_metric_is_not_found(): + attrs = { + "dataset_metadata": {"single_metric": "missing", "ref_data_frequency": "5min"}, + "metrics": [{"id": "a", "data_frequency": "1h"}, {"id": "b"}], + } + + assert get_singlets_recording(frame(attrs)) == Recording(data_frequency="5min") + + +def test_recording_functions_check_their_input(): + with pytest.raises(TypeError): + get_recordings(series({})) + with pytest.raises(TypeError): + get_series_recording(frame({})) + with pytest.raises(TypeError): + get_singlets_recording(series({})) + + +def test_default_aggregation_methods(): + attrs = { + "dataset_metadata": {"metric_key": "id"}, + "metrics": [ + {"id": "avg", "default_aggregation_method": "AVG"}, + {"id": "sum", "default_aggregation_method": "SUM"}, + {"id": "float", "value_dimensions": value_data_type("float")}, + {"id": "integer", "value_dimensions": value_data_type("integer")}, + {"id": "boolean", "value_dimensions": value_data_type("boolean")}, + {"id": "string", "value_dimensions": value_data_type("string")}, + # the method of the metric takes precedence over the default of its data type + { + "id": "boolean_max", + "default_aggregation_method": "MAX", + "value_dimensions": value_data_type("boolean"), + }, + {"id": "null", "default_aggregation_method": None}, + {"id": "shared_data_type", "value_dimensions": []}, + ], + "value_dimensions_shared": [{"column": "value", "value_data_type": "string"}], + } + + methods = get_default_aggregation_methods(frame(attrs)) + + assert methods == { + "avg": "MEAN", + "sum": "SUM", + "float": "MEAN", + "integer": "MEAN", + "boolean": "LAST", + "string": "LAST", + "boolean_max": "MAX", + # the value data type of the shared value dimension "value" applies + "null": "LAST", + "shared_data_type": "LAST", + } + assert methods["metric without metadata"] == "MEAN" + + +@pytest.mark.parametrize( + ("attrs", "method"), + [ + ({}, "MEAN"), + ({"metrics": {"a": {"default_aggregation_method": "SUM"}}}, "MEAN"), + ( + { + "dataset_metadata": {"single_metric": "a"}, + "metrics": [{"id": "a", "value_dimensions": value_data_type("boolean")}], + }, + "LAST", + ), + ({"metrics": [{"id": "a", "default_aggregation_method": "AVG"}]}, "MEAN"), + ], +) +def test_single_metric_default_aggregation_method(attrs, method): + assert get_series_default_aggregation_method(series(attrs)) == method + assert get_singlets_default_aggregation_method(frame(attrs)) == method + + +def utc(*timestamps): + return pd.DatetimeIndex(pd.to_datetime(list(timestamps), utc=True)) + + +def local(index, timezone="Europe/Berlin"): + return [timestamp.isoformat() for timestamp in index.tz_convert(timezone)] + + +@pytest.mark.parametrize( + ("recording", "timestamps", "expected_starts"), + [ + # multiples since 1970-01-01T00:00:00Z plus offset + ( + Recording(data_frequency="15min", data_frequency_offset="5min"), + ["2026-01-01T00:04:59Z", "2026-01-01T00:05:00Z", "2026-01-01T00:19:59Z"], + ["2025-12-31T23:50:00Z", "2026-01-01T00:05:00Z", "2026-01-01T00:05:00Z"], + ), + ( + Recording(data_frequency="1D"), + ["2026-03-29T23:59:00Z"], + ["2026-03-29T00:00:00Z"], + ), + ( + Recording(data_frequency="P2D"), + ["2026-01-04T12:00:00Z"], + ["2026-01-03T00:00:00Z"], + ), + ], +) +def test_recording_interval_starts_in_utc(recording, timestamps, expected_starts): + assert recording_interval_starts(utc(*timestamps), recording).equals(utc(*expected_starts)) + + +def test_recording_intervals_restart_at_local_midnight(): + # the clocks are put forward at 02:00 local time on 2026-03-29, so the day has 23 hours + recording = Recording(data_frequency="2h", data_frequency_timezone="Europe/Berlin") + + grid = recording_grid("2026-03-28T20:00:00Z", "2026-03-29T23:00:00Z", recording) + + assert local(grid) == [ + "2026-03-28T22:00:00+01:00", + "2026-03-29T00:00:00+01:00", + # 2 hours after midnight in elapsed time + "2026-03-29T03:00:00+02:00", + "2026-03-29T05:00:00+02:00", + "2026-03-29T07:00:00+02:00", + "2026-03-29T09:00:00+02:00", + "2026-03-29T11:00:00+02:00", + "2026-03-29T13:00:00+02:00", + "2026-03-29T15:00:00+02:00", + "2026-03-29T17:00:00+02:00", + "2026-03-29T19:00:00+02:00", + "2026-03-29T21:00:00+02:00", + # the last interval of the day lasts 1 hour, until the next local midnight + "2026-03-29T23:00:00+02:00", + "2026-03-30T00:00:00+02:00", + ] + starts = recording_interval_starts( + utc("2026-03-29T21:30:00Z", "2026-03-29T22:00:00Z"), recording + ) + assert local(starts) == ["2026-03-29T23:00:00+02:00", "2026-03-30T00:00:00+02:00"] + + +def test_recording_intervals_not_dividing_a_day(): + recording = Recording(data_frequency="7h", data_frequency_timezone="UTC") + + grid = recording_grid("2026-01-01T00:00:00Z", "2026-01-02T07:00:00Z", recording) + + assert local(grid, "UTC") == [ + "2026-01-01T00:00:00+00:00", + "2026-01-01T07:00:00+00:00", + "2026-01-01T14:00:00+00:00", + "2026-01-01T21:00:00+00:00", + "2026-01-02T00:00:00+00:00", + "2026-01-02T07:00:00+00:00", + ] + + +def test_recording_intervals_with_offset_in_local_time(): + # the intervals of a day start at local midnight plus offset + recording = Recording( + data_frequency="6h", data_frequency_offset="30min", data_frequency_timezone="Asia/Kolkata" + ) + + starts = recording_interval_starts( + utc("2026-01-01T18:30:00Z", "2026-01-01T19:00:00Z"), recording + ) + + assert local(starts, "Asia/Kolkata") == [ + "2026-01-01T18:30:00+05:30", + "2026-01-02T00:30:00+05:30", + ] + + +def test_recording_intervals_of_local_days_and_months(): + # the clocks are put back at 03:00 local time on 2026-10-25, so the day has 25 hours + days = recording_grid( + "2026-10-24T00:00:00Z", + "2026-10-27T00:00:00Z", + Recording(data_frequency="1D", data_frequency_timezone="Europe/Berlin"), + ) + months = recording_grid( + "2026-01-15T00:00:00Z", + "2026-04-02T00:00:00Z", + Recording(data_frequency="MS", data_frequency_timezone="Europe/Berlin"), + ) + + assert local(days) == [ + "2026-10-25T00:00:00+02:00", + "2026-10-26T00:00:00+01:00", + "2026-10-27T00:00:00+01:00", + ] + assert local(months) == [ + "2026-02-01T00:00:00+01:00", + "2026-03-01T00:00:00+01:00", + "2026-04-01T00:00:00+02:00", + ] + + +def test_recording_interval_starts_of_series_and_naive_timestamps(): + timestamps = pd.Series( + pd.to_datetime(["2026-01-01T00:07:00", "2026-01-01T00:29:00"]), index=["a", "b"] + ) + + starts = recording_interval_starts(timestamps, Recording(data_frequency="15min")) + + assert starts.index.to_list() == ["a", "b"] + assert starts.to_list() == list(utc("2026-01-01T00:00:00Z", "2026-01-01T00:15:00Z")) + + +def test_recording_grid_from_start_to_end(): + recording = Recording(data_frequency="15min") + + assert recording_grid("2026-01-01T00:15:00Z", "2026-01-01T00:30:00Z", recording).equals( + utc("2026-01-01T00:15:00Z", "2026-01-01T00:30:00Z") + ) + assert len(recording_grid("2026-01-01T00:01:00Z", "2026-01-01T00:14:00Z", recording)) == 0 + assert len(recording_grid("2026-01-02T00:00:00Z", "2026-01-01T00:00:00Z", recording)) == 0 + + +@pytest.mark.parametrize( + "recording", + [ + Recording(recording_mode="periodic"), + Recording(data_frequency="36h", data_frequency_timezone="Europe/Berlin"), + ], +) +def test_recording_without_usable_frequency(recording): + with pytest.raises(ValueError, match="data_frequency|calendar"): + recording_interval_starts(utc("2026-01-01T00:00:00Z"), recording) From 4cfc58fff9863a4a4b9f29f7a215f59cc513859a Mon Sep 17 00:00:00 2001 From: stewit <> Date: Mon, 26 Oct 2026 13:19:00 +0100 Subject: [PATCH 12/15] cleanup --- CHANGELOG.md | 4 +- src/hdhelpers/metadata/__init__.py | 6 +- src/hdhelpers/metadata/_helpers.py | 4 +- src/hdhelpers/metadata/_recording.py | 164 ++---------------- src/hdhelpers/metadata/helpers.py | 99 +++-------- tests/metadata/test_metadata_recording.py | 156 +---------------- .../test_metadata_without_metadata.py | 51 ++++++ 7 files changed, 100 insertions(+), 384 deletions(-) create mode 100644 tests/metadata/test_metadata_without_metadata.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 690a341..2f6cc58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,9 @@ - entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error - new functions get_default_timezones, get_series_default_timezone and get_singlets_default_timezone provide the default time zone of the metrics ("default_timezone" of the current metadata convention), falling back to the one of the dataset metadata. It is meant for computations, e.g. bucketing days when aggregating, not for displaying timestamps, which is what the time zone of the plot target settings is for. New section "Which time zone to compute in?" in the documentation - documentation: the information requested from get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info is interpreted as a glom spec. New section "How to query nested information?" with examples for dotted paths into sub dictionaries, other glom specs and keys containing dots -- new class Recording and functions get_recordings, get_series_recording and get_singlets_recording provide how the values of the metrics are recorded: "recording_mode", "data_frequency" with "data_frequency_offset" and "data_frequency_timezone", and "recording_aggregation_method" of the current metadata convention. The frequency falls back to "ref_data_frequency", "ref_data_frequency_offset" and "ref_data_frequency_timezone" of the dataset metadata as a whole, i.e. offset and time zone only apply together with the frequency of the same level. A missing offset means 0, a missing time zone UTC -- new functions recording_interval_starts and recording_grid provide the starts of the recording intervals of a Recording: multiples of the frequency since 1970-01-01T00:00:00Z plus the offset, or with a time zone, intervals restarting at each local midnight (the last one of a day cut at the next midnight, e.g. on days with a daylight saving time change), local calendar days or local calendar months +- new class Recording and functions get_recordings, get_series_recording and get_singlets_recording provide how the values of the metrics are recorded: "recording_mode", "data_frequency" with "data_frequency_offset" and "data_frequency_timezone", and "recording_aggregation_method" of the current metadata convention. The frequency falls back to "ref_data_frequency", "ref_data_frequency_offset" and "ref_data_frequency_timezone" of the dataset metadata as a whole, i.e. offset and time zone only apply together with the frequency of the same level. The values are provided as given in the metadata, the documentation of Recording explains what they mean - new functions get_default_aggregation_methods, get_series_default_aggregation_method and get_singlets_default_aggregation_method provide the default aggregation method of the metrics ("default_aggregation_method"), normalizing "AVG" to "MEAN". Without one, it is "LAST" for metrics whose value dimension "value" has the value data type "boolean" or "string", otherwise "MEAN" +- new function without_metadata provides a copy of a Series or DataFrame without its metadata (attrs) to compute on. pandas deep-copies the attrs to every object an operation creates, which makes e.g. loops over the metrics of a MultiTSFrame with the metadata of many metrics slow. The copy shares the data until it is changed ## [0.0.9] - 2026-09-10 - singletsframe support diff --git a/src/hdhelpers/metadata/__init__.py b/src/hdhelpers/metadata/__init__.py index 0a1d1a9..969d034 100644 --- a/src/hdhelpers/metadata/__init__.py +++ b/src/hdhelpers/metadata/__init__.py @@ -10,8 +10,6 @@ get_series_recording, get_singlets_default_aggregation_method, get_singlets_recording, - recording_grid, - recording_interval_starts, get_default_timezones, get_display_names, get_measurements, @@ -35,6 +33,7 @@ get_singlets_short_display_names, get_singlets_units, get_units, + without_metadata, ) __all__ = [ @@ -65,9 +64,8 @@ "get_recordings", "get_series_recording", "get_singlets_recording", - "recording_interval_starts", - "recording_grid", "get_default_aggregation_methods", "get_series_default_aggregation_method", "get_singlets_default_aggregation_method", + "without_metadata", ] diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 37016fc..7af114d 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -316,8 +316,8 @@ def _default_timezones( def default_timezone_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str | None]: """The default time zone of each metric, falling back to the one of the dataset metadata - As specified by the metadata conventions, the default time zone of a metric only falls back - to the one of the dataset metadata, not to its value dimensions or inherited properties. + The default time zone of a metric only falls back to the one of the dataset metadata, not to + its value dimensions or inherited properties. The default of the returned defaultdict is the one of the dataset metadata, since it applies to all metrics without their own. Values which are no (non-empty) strings count as missing. """ diff --git a/src/hdhelpers/metadata/_recording.py b/src/hdhelpers/metadata/_recording.py index 4064196..5ae2dc3 100644 --- a/src/hdhelpers/metadata/_recording.py +++ b/src/hdhelpers/metadata/_recording.py @@ -3,21 +3,12 @@ The frequency of the recorded values of a metric falls back as a whole: If the metric has a "data_frequency", it applies together with the metric's "data_frequency_offset" and "data_frequency_timezone". Otherwise "ref_data_frequency" of the dataset metadata applies -together with "ref_data_frequency_offset" and "ref_data_frequency_timezone". A missing offset -is 0. - -The recording intervals start at the multiples of the frequency plus the offset. Without time -zone these are the multiples since 1970-01-01T00:00:00Z. With time zone and a frequency shorter -than a day, the intervals restart at each local midnight (plus offset) of the time zone, counted -in elapsed time, and the last interval of a day ends at the next local midnight. With time zone -and a frequency of a day or longer, the intervals follow the calendar of the time zone, e.g. -local days of 23, 24 or 25 hours. +together with "ref_data_frequency_offset" and "ref_data_frequency_timezone". """ -import math from collections import defaultdict from dataclasses import dataclass -from typing import Any, TypeIs, cast +from typing import Any, cast import pandas as pd @@ -36,16 +27,22 @@ _DEFAULT_AGGREGATION_METHOD = "MEAN" _AGGREGATION_METHOD_ALIASES = {"AVG": "MEAN"} -_DAY = pd.Timedelta(days=1) - @dataclass(frozen=True) class Recording: - """How the values of a metric are recorded + """How the values of a metric are recorded, as given in its metadata + + The fields mean the following: - The fields are taken from the metadata as given. data_frequency_offset and - data_frequency_timezone belong to data_frequency: They are None if no data_frequency - applies. A data_frequency_offset of None means 0. + Attributes: + recording_mode (str | None): "periodic": one value per interval of data_frequency, timestamped with the start of the interval. A missing value is a gap. "on_change": a value is recorded when it changes by more than the deadband of its value dimension and holds until the next one. Missing timestamps are no gaps. "irregular": values at arbitrary points in time. None if unknown. + data_frequency (str | None): Frequency as Pandas date offset alias or timedelta string, e.g. "15min", "1D" or "1MS". For "on_change", the interval after which the value is recorded again at the latest, also without a change (heartbeat). + data_frequency_offset (str | None): Timedelta string by which the intervals of data_frequency are shifted, e.g. "5min". None means 0. + data_frequency_timezone (str | None): Time zone the intervals of data_frequency are aligned to, e.g. "Europe/Berlin". None means UTC. + recording_aggregation_method (str | None): Only for "periodic": the method each value aggregates its interval with, e.g. "MEAN" or "MAX". None if the values are samples, e.g. the value holding at the timestamp. + + data_frequency_offset and data_frequency_timezone belong to data_frequency: They are None + if no data_frequency applies. """ recording_mode: str | None = None @@ -170,136 +167,3 @@ def single_metric_default_aggregation_method(data: pd.DataFrame | pd.Series) -> empty=_DEFAULT_AGGREGATION_METHOD, ), ) - - -def _grid_parameters( - recording: Recording, -) -> tuple[pd.offsets.BaseOffset, pd.Timedelta, str | None]: - """Frequency as Pandas offset, offset as Timedelta and time zone of a recording""" - if recording.data_frequency is None: - raise ValueError("The recording has no data_frequency, which defines its intervals.") - try: - frequency = pd.tseries.frequencies.to_offset(recording.data_frequency) - except ValueError: - # e.g. "P1D" or "1 days" - frequency = pd.tseries.frequencies.to_offset( - pd.to_timedelta(recording.data_frequency) # type: ignore[call-overload] - ) - offset = pd.to_timedelta(recording.data_frequency_offset or "0s") - return frequency, offset, recording.data_frequency_timezone - - -def _is_shorter_than_a_day(frequency: pd.offsets.BaseOffset) -> TypeIs[pd.offsets.Tick]: - return isinstance(frequency, pd.offsets.Tick) and pd.Timedelta(frequency) < _DAY - - -def _utc_series(timestamps: pd.DatetimeIndex | pd.Series) -> pd.Series: - """Timestamps as Series in UTC, naive timestamps are considered to be UTC""" - series = pd.Series(timestamps) if isinstance(timestamps, pd.DatetimeIndex) else timestamps - if series.dt.tz is None: - return series.dt.tz_localize("UTC") - return series.dt.tz_convert("UTC") - - -def _localize_midnights(naive_midnights: pd.Series, timezone: str) -> pd.Series: - """Localize local midnights, shifting nonexistent ones forward and preferring DST""" - return naive_midnights.dt.tz_localize( - timezone, ambiguous=[True] * len(naive_midnights), nonexistent="shift_forward" - ) - - -def _calendar_interval_starts( - utc: pd.Series, frequency: pd.offsets.BaseOffset, offset: pd.Timedelta, timezone: str -) -> pd.Series: - """Interval starts of a frequency of a day or longer in the calendar of the time zone""" - local_days = (utc - offset).dt.tz_convert(timezone).dt.tz_localize(None).dt.normalize() - if isinstance(frequency, pd.offsets.Day | pd.offsets.Tick): - step = ( - frequency.n * _DAY if isinstance(frequency, pd.offsets.Day) else pd.Timedelta(frequency) - ) - if step % _DAY != pd.Timedelta(0): - raise ValueError( - f'A frequency of "{frequency.freqstr}" cannot follow the calendar of a time zone,' - " only multiples of a day can." - ) - # multiples of whole days since 1970-01-01 in local time - epoch = pd.Timestamp("1970-01-01") - starts = epoch + ((local_days - epoch) // step) * step - else: - # e.g. the first day of the month for "MS" - unique_starts = {day: frequency.rollback(day) for day in local_days.dropna().unique()} - starts = local_days.map(unique_starts).astype(local_days.dtype) - return _localize_midnights(starts, timezone) + offset - - -def recording_interval_starts[T: (pd.DatetimeIndex, pd.Series)]( - timestamps: T, recording: Recording -) -> T: - """The start of the recording interval of each timestamp, in UTC""" - frequency, offset, timezone = _grid_parameters(recording) - utc = _utc_series(timestamps) - - if timezone is None and isinstance(frequency, pd.offsets.Tick): - epoch = pd.Timestamp(0, tz="UTC") - duration = pd.Timedelta(frequency) - starts = epoch + offset + ((utc - epoch - offset) // duration) * duration - elif timezone is not None and _is_shorter_than_a_day(frequency): - duration = pd.Timedelta(frequency) - # the intervals of a day start at its local midnight plus offset - local_midnights = (utc - offset).dt.tz_convert(timezone).dt.normalize() - day_starts = local_midnights.dt.tz_convert("UTC") + offset - starts = day_starts + ((utc - day_starts) // duration) * duration - else: - starts = _calendar_interval_starts(utc, frequency, offset, timezone or "UTC").dt.tz_convert( - "UTC" - ) - - if isinstance(timestamps, pd.DatetimeIndex): - return pd.DatetimeIndex(starts.to_numpy(), name=timestamps.name) - return starts.set_axis(timestamps.index) - - -def recording_grid( - start: pd.Timestamp, end: pd.Timestamp, recording: Recording -) -> pd.DatetimeIndex: - """The recording interval starts from start to end, both inclusive, in UTC""" - frequency, offset, timezone = _grid_parameters(recording) - start = _utc_series(pd.DatetimeIndex([start])).iloc[0] - end = _utc_series(pd.DatetimeIndex([end])).iloc[0] - if start > end: - return pd.DatetimeIndex([], tz="UTC") - first = recording_interval_starts(pd.DatetimeIndex([start]), recording)[0] - - if timezone is None and isinstance(frequency, pd.offsets.Tick): - grid = pd.date_range(first, end, freq=pd.Timedelta(frequency)) - elif timezone is not None and _is_shorter_than_a_day(frequency): - duration = pd.Timedelta(frequency) - local_midnights = pd.date_range( - (first - offset).tz_convert(timezone).normalize(), - (end - offset).tz_convert(timezone).normalize() + _DAY, - freq="D", - ) - day_starts = local_midnights.tz_convert("UTC") + offset - grid = pd.DatetimeIndex( - [ - day_start + index * duration - for day_start, next_day_start in zip(day_starts[:-1], day_starts[1:], strict=True) - for index in range(math.ceil((next_day_start - day_start) / duration)) - ], - tz="UTC", - ) - else: - # step through the calendar of the time zone via the interval starts of the days - days = pd.date_range( - (first - offset).tz_convert(timezone or "UTC").normalize().tz_localize(None), - (end - offset).tz_convert(timezone or "UTC").normalize().tz_localize(None), - freq="D", - ) - candidates = ( - _localize_midnights(pd.Series(days), timezone or "UTC").dt.tz_convert("UTC") + offset - ) - grid = pd.DatetimeIndex( - recording_interval_starts(pd.DatetimeIndex(candidates), recording).unique() - ) - - return grid[(grid >= start) & (grid <= end)] diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index d494c8a..0d68e19 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -24,10 +24,6 @@ single_metric_default_aggregation_method, single_metric_recording, ) -from hdhelpers.metadata._recording import recording_grid as _recording_grid -from hdhelpers.metadata._recording import ( - recording_interval_starts as _recording_interval_starts, -) from hdhelpers.metadata._helpers import ( check_dataframe, check_series, @@ -1044,8 +1040,8 @@ def get_default_timezones(multitsframe: pd.DataFrame) -> defaultdict[str, str | "+02:00", both accepted by pandas, e.g. by ``tz_convert``. The default time zone of a metric falls back to the one of the dataset metadata, which - applies to all metrics without their own. As specified by the metadata conventions, it does - not fall back to value dimensions or inherited properties. + applies to all metrics without their own. It does not fall back to value dimensions or + inherited properties. Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. @@ -1165,14 +1161,13 @@ def get_recordings(multitsframe: pd.DataFrame) -> defaultdict[str, Recording]: The recording consists of the "recording_mode" of the metric (e.g. "periodic" or "on_change"), the frequency of its values with offset and time zone and its - "recording_aggregation_method", see "Recording of the values" in the metadata conventions. + "recording_aggregation_method", see :class:`Recording` for what they mean. The frequency falls back as a whole: If the metric has a "data_frequency", it applies together with the metric's "data_frequency_offset" and "data_frequency_timezone". Otherwise "ref_data_frequency" of the dataset metadata applies together with "ref_data_frequency_offset" and "ref_data_frequency_timezone". The values are provided as - given in the metadata, an offset of None means 0. Use :func:`recording_interval_starts` and - :func:`recording_grid` for the timestamps of the recording intervals. + given in the metadata. Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. @@ -1291,8 +1286,7 @@ def get_default_aggregation_methods(multitsframe: pd.DataFrame) -> defaultdict[s of the metric, with "AVG" normalized to its alias "MEAN". If the metric does not provide one, it is "LAST" for values of the data type "boolean" or "string" (the "value_data_type" of the value dimension "value"), i.e. the state holding at the end of the interval, and "MEAN" - otherwise. As specified by the metadata conventions, it is not taken from inherited - properties. + otherwise. It is not taken from inherited properties. Args: multitsframe (pd.DataFrame): MTSF with metadata following the convention. @@ -1394,74 +1388,37 @@ def get_singlets_default_aggregation_method(singletsframe: pd.DataFrame) -> str: return single_metric_default_aggregation_method(singletsframe) -def recording_interval_starts[T: (pd.DatetimeIndex, pd.Series)]( - timestamps: T, recording: Recording -) -> T: - """Gets the start of the recording interval of each timestamp +def without_metadata[T: (pd.Series, pd.DataFrame)](data: T) -> T: + """Gets a copy of a Series or DataFrame without its metadata, to compute on - The recording intervals start at the multiples of the frequency of the recording plus its - offset (0 if None): + pandas deep-copies the metadata (``attrs``) of a Series or DataFrame to every object an + operation creates from it, e.g. when filtering rows, selecting a column or grouping. For + metadata of many metrics, e.g. with comments, loops over the metrics then take much longer + than the computation itself. Compute on the copy without metadata instead and attach the + metadata of the result once at the end. - * Without time zone, these are the multiples since 1970-01-01T00:00:00Z. - * With time zone and a frequency shorter than a day, the intervals restart at each local - midnight (plus offset) of the time zone, counted in elapsed time. The last interval of a - day ends at the next local midnight, so it is shorter on days with a daylight saving time - change or if the frequency does not divide a day. This is how e.g. the hetida platform - compresses values in windows. - * With time zone and a frequency of a day or longer, e.g. "1D" or "1MS", the intervals follow - the calendar of the time zone, e.g. local days of 23, 24 or 25 hours. + The copy is cheap: It shares the data with ``data`` until one of them is changed (copy on + write), so changing the copy does not change ``data``. Args: - timestamps (pd.DatetimeIndex | pd.Series): The timestamps, naive ones are considered to be UTC. - recording (Recording): The recording, e.g. from :func:`get_series_recording`. + data (pd.Series | pd.DataFrame): Series or DataFrame, e.g. a MTSF with metadata. Returns: - pd.DatetimeIndex | pd.Series: The start of the recording interval of each timestamp in UTC, as DatetimeIndex or Series (with the index of `timestamps`) like `timestamps`. - - Raises: - ValueError: If the recording has no data_frequency or its frequency, offset or time zone cannot be interpreted. + pd.Series | pd.DataFrame: Copy of ``data`` with empty ``attrs``. Code example: - .. doctest:: metadata.recording_interval_starts + .. doctest:: metadata.without_metadata - >>> recording = Recording(data_frequency="2h", data_frequency_timezone="Europe/Berlin") - >>> # on 2026-03-29 the clocks in Berlin are put forward at 2:00 local time (1:00 UTC) - >>> starts = recording_interval_starts( - ... pd.DatetimeIndex(["2026-03-29T00:30:00Z", "2026-03-29T01:30:00Z"]), recording - ... ) - >>> [start.isoformat() for start in starts.tz_convert("Europe/Berlin")] - ['2026-03-29T00:00:00+01:00', '2026-03-29T03:00:00+02:00'] + >>> multitsframe.attrs = {"dataset_metadata": {"metric_key": "id"}, "metrics": []} + >>> data = without_metadata(multitsframe) + >>> data.attrs + {} + >>> multitsframe.attrs + {'dataset_metadata': {'metric_key': 'id'}, 'metrics': []} """ - return _recording_interval_starts(timestamps, recording) - - -def recording_grid( - start: pd.Timestamp | str, end: pd.Timestamp | str, recording: Recording -) -> pd.DatetimeIndex: - """Gets the starts of the recording intervals from start to end, both inclusive - - For a periodic recording, these are the timestamps at which values are expected. See - :func:`recording_interval_starts` for how the intervals are determined. - - Args: - start (pd.Timestamp | str): Start, naive timestamps are considered to be UTC. - end (pd.Timestamp | str): End, naive timestamps are considered to be UTC. - recording (Recording): The recording, e.g. from :func:`get_series_recording`. - - Returns: - pd.DatetimeIndex: The interval starts in UTC. - - Raises: - ValueError: If the recording has no data_frequency or its frequency, offset or time zone cannot be interpreted. - - Code example: - - .. doctest:: metadata.recording_grid - - >>> recording = Recording(data_frequency="15min", data_frequency_offset="5min") - >>> grid = recording_grid("2026-01-01T00:00:00Z", "2026-01-01T00:40:00Z", recording) - >>> [timestamp.isoformat() for timestamp in grid] - ['2026-01-01T00:05:00+00:00', '2026-01-01T00:20:00+00:00', '2026-01-01T00:35:00+00:00'] - """ - return _recording_grid(pd.Timestamp(start), pd.Timestamp(end), recording) + check_series_or_dataframe(data) + # The constructor does not copy attrs, in contrast to e.g. copy(deep=False) + copy = type(data)(data) + copy.attrs = {} + return copy diff --git a/tests/metadata/test_metadata_recording.py b/tests/metadata/test_metadata_recording.py index 628b7b5..527bac6 100644 --- a/tests/metadata/test_metadata_recording.py +++ b/tests/metadata/test_metadata_recording.py @@ -1,4 +1,4 @@ -"""Recording of the metrics, their default aggregation methods and the recording intervals""" +"""Recording of the metrics and their default aggregation methods""" import pandas as pd import pytest @@ -11,8 +11,6 @@ get_series_recording, get_singlets_default_aggregation_method, get_singlets_recording, - recording_grid, - recording_interval_starts, ) @@ -183,155 +181,3 @@ def test_default_aggregation_methods(): def test_single_metric_default_aggregation_method(attrs, method): assert get_series_default_aggregation_method(series(attrs)) == method assert get_singlets_default_aggregation_method(frame(attrs)) == method - - -def utc(*timestamps): - return pd.DatetimeIndex(pd.to_datetime(list(timestamps), utc=True)) - - -def local(index, timezone="Europe/Berlin"): - return [timestamp.isoformat() for timestamp in index.tz_convert(timezone)] - - -@pytest.mark.parametrize( - ("recording", "timestamps", "expected_starts"), - [ - # multiples since 1970-01-01T00:00:00Z plus offset - ( - Recording(data_frequency="15min", data_frequency_offset="5min"), - ["2026-01-01T00:04:59Z", "2026-01-01T00:05:00Z", "2026-01-01T00:19:59Z"], - ["2025-12-31T23:50:00Z", "2026-01-01T00:05:00Z", "2026-01-01T00:05:00Z"], - ), - ( - Recording(data_frequency="1D"), - ["2026-03-29T23:59:00Z"], - ["2026-03-29T00:00:00Z"], - ), - ( - Recording(data_frequency="P2D"), - ["2026-01-04T12:00:00Z"], - ["2026-01-03T00:00:00Z"], - ), - ], -) -def test_recording_interval_starts_in_utc(recording, timestamps, expected_starts): - assert recording_interval_starts(utc(*timestamps), recording).equals(utc(*expected_starts)) - - -def test_recording_intervals_restart_at_local_midnight(): - # the clocks are put forward at 02:00 local time on 2026-03-29, so the day has 23 hours - recording = Recording(data_frequency="2h", data_frequency_timezone="Europe/Berlin") - - grid = recording_grid("2026-03-28T20:00:00Z", "2026-03-29T23:00:00Z", recording) - - assert local(grid) == [ - "2026-03-28T22:00:00+01:00", - "2026-03-29T00:00:00+01:00", - # 2 hours after midnight in elapsed time - "2026-03-29T03:00:00+02:00", - "2026-03-29T05:00:00+02:00", - "2026-03-29T07:00:00+02:00", - "2026-03-29T09:00:00+02:00", - "2026-03-29T11:00:00+02:00", - "2026-03-29T13:00:00+02:00", - "2026-03-29T15:00:00+02:00", - "2026-03-29T17:00:00+02:00", - "2026-03-29T19:00:00+02:00", - "2026-03-29T21:00:00+02:00", - # the last interval of the day lasts 1 hour, until the next local midnight - "2026-03-29T23:00:00+02:00", - "2026-03-30T00:00:00+02:00", - ] - starts = recording_interval_starts( - utc("2026-03-29T21:30:00Z", "2026-03-29T22:00:00Z"), recording - ) - assert local(starts) == ["2026-03-29T23:00:00+02:00", "2026-03-30T00:00:00+02:00"] - - -def test_recording_intervals_not_dividing_a_day(): - recording = Recording(data_frequency="7h", data_frequency_timezone="UTC") - - grid = recording_grid("2026-01-01T00:00:00Z", "2026-01-02T07:00:00Z", recording) - - assert local(grid, "UTC") == [ - "2026-01-01T00:00:00+00:00", - "2026-01-01T07:00:00+00:00", - "2026-01-01T14:00:00+00:00", - "2026-01-01T21:00:00+00:00", - "2026-01-02T00:00:00+00:00", - "2026-01-02T07:00:00+00:00", - ] - - -def test_recording_intervals_with_offset_in_local_time(): - # the intervals of a day start at local midnight plus offset - recording = Recording( - data_frequency="6h", data_frequency_offset="30min", data_frequency_timezone="Asia/Kolkata" - ) - - starts = recording_interval_starts( - utc("2026-01-01T18:30:00Z", "2026-01-01T19:00:00Z"), recording - ) - - assert local(starts, "Asia/Kolkata") == [ - "2026-01-01T18:30:00+05:30", - "2026-01-02T00:30:00+05:30", - ] - - -def test_recording_intervals_of_local_days_and_months(): - # the clocks are put back at 03:00 local time on 2026-10-25, so the day has 25 hours - days = recording_grid( - "2026-10-24T00:00:00Z", - "2026-10-27T00:00:00Z", - Recording(data_frequency="1D", data_frequency_timezone="Europe/Berlin"), - ) - months = recording_grid( - "2026-01-15T00:00:00Z", - "2026-04-02T00:00:00Z", - Recording(data_frequency="MS", data_frequency_timezone="Europe/Berlin"), - ) - - assert local(days) == [ - "2026-10-25T00:00:00+02:00", - "2026-10-26T00:00:00+01:00", - "2026-10-27T00:00:00+01:00", - ] - assert local(months) == [ - "2026-02-01T00:00:00+01:00", - "2026-03-01T00:00:00+01:00", - "2026-04-01T00:00:00+02:00", - ] - - -def test_recording_interval_starts_of_series_and_naive_timestamps(): - timestamps = pd.Series( - pd.to_datetime(["2026-01-01T00:07:00", "2026-01-01T00:29:00"]), index=["a", "b"] - ) - - starts = recording_interval_starts(timestamps, Recording(data_frequency="15min")) - - assert starts.index.to_list() == ["a", "b"] - assert starts.to_list() == list(utc("2026-01-01T00:00:00Z", "2026-01-01T00:15:00Z")) - - -def test_recording_grid_from_start_to_end(): - recording = Recording(data_frequency="15min") - - assert recording_grid("2026-01-01T00:15:00Z", "2026-01-01T00:30:00Z", recording).equals( - utc("2026-01-01T00:15:00Z", "2026-01-01T00:30:00Z") - ) - assert len(recording_grid("2026-01-01T00:01:00Z", "2026-01-01T00:14:00Z", recording)) == 0 - assert len(recording_grid("2026-01-02T00:00:00Z", "2026-01-01T00:00:00Z", recording)) == 0 - - -@pytest.mark.parametrize( - "recording", - [ - Recording(recording_mode="periodic"), - Recording(data_frequency="36h", data_frequency_timezone="Europe/Berlin"), - ], -) -def test_recording_without_usable_frequency(recording): - with pytest.raises(ValueError, match="data_frequency|calendar"): - recording_interval_starts(utc("2026-01-01T00:00:00Z"), recording) diff --git a/tests/metadata/test_metadata_without_metadata.py b/tests/metadata/test_metadata_without_metadata.py new file mode 100644 index 0000000..3a2c344 --- /dev/null +++ b/tests/metadata/test_metadata_without_metadata.py @@ -0,0 +1,51 @@ +import pandas as pd +import pytest + +from hdhelpers.metadata import without_metadata + + +class NotCopyable: + """Metadata which fails if it is copied""" + + def __deepcopy__(self, memo): + raise AssertionError("The metadata was copied") + + +@pytest.mark.parametrize( + "data", + [ + pd.DataFrame({"metric": ["a", "b"], "value": [1.0, 2.0]}), + pd.Series([1.0, 2.0], index=pd.date_range("2026-01-01", periods=2, tz="UTC"), name="a"), + ], +) +def test_without_metadata(data): + metadata = {"metrics": [{"id": "a"}], "not copyable": NotCopyable()} + data.attrs = metadata + + copy = without_metadata(data) + + assert copy.attrs == {} + assert data.attrs == metadata + # comparing the data creates objects from it, which would copy its metadata + data.attrs = {} + pd.testing.assert_index_equal(copy.index, data.index) + if isinstance(data, pd.Series): + assert copy.name == data.name + pd.testing.assert_series_equal(copy, data, check_flags=False) + else: + pd.testing.assert_frame_equal(copy, data, check_flags=False) + + +def test_changing_the_copy_does_not_change_the_data(): + data = pd.DataFrame({"value": [1.0, 2.0]}) + data.attrs = {"metrics": []} + + copy = without_metadata(data) + copy.loc[0, "value"] = -1.0 + + assert data.loc[0, "value"] == 1.0 + + +def test_without_metadata_requires_series_or_dataframe(): + with pytest.raises(TypeError): + without_metadata([1.0, 2.0]) From b81c995c6fa003d8ada25ca65629b7b6de47208f Mon Sep 17 00:00:00 2001 From: jenny kupzig Date: Tue, 27 Oct 2026 08:47:13 +0100 Subject: [PATCH 13/15] update docs --- docs/genindex.html | 46 +++++- docs/index.html | 365 +++++++++++++++++++++++++++++++++++++++++++- docs/objects.inv | Bin 540 -> 663 bytes docs/searchindex.js | 2 +- 4 files changed, 409 insertions(+), 4 deletions(-) diff --git a/docs/genindex.html b/docs/genindex.html index ce8be7b..42823ec 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -39,17 +39,35 @@

        Navigation

        Index

        - G + D + | G | H | I | M | R | S + | W
        +

        D

        + + + +
        +

        G

          +
        • get_series_recording() (in module hdhelpers.metadata) +
        • get_series_short_display_name() (in module hdhelpers.metadata)
        • get_series_unit() (in module hdhelpers.metadata)
        • get_short_display_names() (in module hdhelpers.metadata) +
          • +
        • get_singlets_default_aggregation_method() (in module hdhelpers.metadata)
        • get_singlets_default_timezone() (in module hdhelpers.metadata)
          • @@ -93,6 +119,8 @@

          G

        • get_singlets_metric_info() (in module hdhelpers.metadata)
        • get_singlets_names() (in module hdhelpers.metadata) +
          • +
        • get_singlets_recording() (in module hdhelpers.metadata)
        • get_singlets_short_display_names() (in module hdhelpers.metadata)
          • @@ -173,6 +201,14 @@

          M

          R

          + @@ -190,6 +226,14 @@

          S

          +

          W

          + + +
          +
          diff --git a/docs/index.html b/docs/index.html index 2b4a729..f303374 100644 --- a/docs/index.html +++ b/docs/index.html @@ -75,6 +75,126 @@

          Functions +
          +class hdhelpers.metadata.Recording(recording_mode=None, data_frequency=None, data_frequency_offset=None, data_frequency_timezone=None, recording_aggregation_method=None)
          +

          Bases: object

          +

          How the values of a metric are recorded, as given in its metadata

          +

          The fields mean the following:

          +
          +
          Parameters:
          +
            +

          • recording_mode (str | None)

            • +

          • data_frequency (str | None)

            • +

          • data_frequency_offset (str | None)

            • +

          • data_frequency_timezone (str | None)

            • +

          • recording_aggregation_method (str | None)

            • +
          +
          +
          +
          +
          +recording_mode
          +

          “periodic”: one value per interval of data_frequency, timestamped with the start of the interval. A missing value is a gap. “on_change”: a value is recorded when it changes by more than the deadband of its value dimension and holds until the next one. Missing timestamps are no gaps. “irregular”: values at arbitrary points in time. None if unknown.

          +
          +
          Type:
          +

          str | None

          +
          +
          +
          + +
          +
          +data_frequency
          +

          Frequency as Pandas date offset alias or timedelta string, e.g. “15min”, “1D” or “1MS”. For “on_change”, the interval after which the value is recorded again at the latest, also without a change (heartbeat).

          +
          +
          Type:
          +

          str | None

          +
          +
          +
          + +
          +
          +data_frequency_offset
          +

          Timedelta string by which the intervals of data_frequency are shifted, e.g. “5min”. None means 0.

          +
          +
          Type:
          +

          str | None

          +
          +
          +
          + +
          +
          +data_frequency_timezone
          +

          Time zone the intervals of data_frequency are aligned to, e.g. “Europe/Berlin”. None means UTC.

          +
          +
          Type:
          +

          str | None

          +
          +
          +
          + +
          +
          +recording_aggregation_method
          +

          Only for “periodic”: the method each value aggregates its interval with, e.g. “MEAN” or “MAX”. None if the values are samples, e.g. the value holding at the timestamp.

          +
          +
          Type:
          +

          str | None

          +
          +
          +
          + +

          data_frequency_offset and data_frequency_timezone belong to data_frequency: They are None +if no data_frequency applies.

          +
          + +
          +
          +hdhelpers.metadata.get_default_aggregation_methods(multitsframe)
          +

          Gets the method each metric of a MultiTSFrame should be aggregated with by default

          +

          E.g. for plotting or reporting it in coarser intervals. It is the “default_aggregation_method” +of the metric, with “AVG” normalized to its alias “MEAN”. If the metric does not provide one, +it is “LAST” for values of the data type “boolean” or “string” (the “value_data_type” of the +value dimension “value”), i.e. the state holding at the end of the interval, and “MEAN” +otherwise. It is not taken from inherited properties.

          +
          +
          Parameters:
          +

          multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

          +
          +
          Returns:
          +

          Dictionary, where keys are the entries of the metrics metadata specified via “metric_key” in “dataset_metadata” and values are their default aggregation methods, e.g. “MEAN”, “SUM” or “LAST”. Defaults to “MEAN” for metrics without metadata.

          +
          +
          Return type:
          +

          defaultdict[str, str]

          +
          +
          Raises:
          +

          TypeError – If multitsframe is not a DataFrame.

          +
          +
          +

          Code example:

          +
          >>> multitsframe.attrs = {
+...     "dataset_metadata": {"metric_key": "id"},
+...     "metrics": [
+...         {"id": "level", "default_aggregation_method": "AVG"},
+...         {"id": "rainfall", "default_aggregation_method": "SUM"},
+...         {
+...             "id": "pump_running",
+...             "value_dimensions": [{"column": "value", "value_data_type": "boolean"}],
+...         },
+...     ],
+... }
+>>> methods = get_default_aggregation_methods(multitsframe)
+>>> methods["level"], methods["rainfall"], methods["pump_running"]
+('MEAN', 'SUM', 'LAST')
+>>> methods["metric_without_metadata"]
+'MEAN'
+
          +
          +
          +
          hdhelpers.metadata.get_default_timezones(multitsframe)
          @@ -88,8 +208,8 @@

          Functionstz_convert.

          The default time zone of a metric falls back to the one of the dataset metadata, which -applies to all metrics without their own. As specified by the metadata conventions, it does -not fall back to value dimensions or inherited properties.

          +applies to all metrics without their own. It does not fall back to value dimensions or +inherited properties.

          Parameters:

          multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

          @@ -372,6 +492,89 @@

          Functions +
          +hdhelpers.metadata.get_recordings(multitsframe)
          +

          Gets how the values of each metric of a MultiTSFrame are recorded

          +

          The recording consists of the “recording_mode” of the metric (e.g. “periodic” or +“on_change”), the frequency of its values with offset and time zone and its +“recording_aggregation_method”, see Recording for what they mean.

          +

          The frequency falls back as a whole: If the metric has a “data_frequency”, it applies +together with the metric’s “data_frequency_offset” and “data_frequency_timezone”. Otherwise +“ref_data_frequency” of the dataset metadata applies together with +“ref_data_frequency_offset” and “ref_data_frequency_timezone”. The values are provided as +given in the metadata.

          +
          +
          Parameters:
          +

          multitsframe (pd.DataFrame) – MTSF with metadata following the convention.

          +
          +
          Returns:
          +

          Dictionary, where keys are the entries of the metrics metadata specified via “metric_key” in “dataset_metadata” and values are their recordings. Defaults to a recording with the frequency of the dataset metadata, also for metrics without metadata.

          +
          +
          Return type:
          +

          defaultdict[str, Recording]

          +
          +
          Raises:
          +

          TypeError – If multitsframe is not a DataFrame.

          +
          +
          +

          Code example:

          +
          >>> multitsframe.attrs = {
+...     "dataset_metadata": {"metric_key": "id", "ref_data_frequency": "1h"},
+...     "metrics": [
+...         {
+...             "id": "average_level",
+...             "recording_mode": "periodic",
+...             "data_frequency": "15min",
+...             "recording_aggregation_method": "MEAN",
+...         },
+...         {"id": "pump_state", "recording_mode": "on_change"},
+...     ],
+... }
+>>> recordings = get_recordings(multitsframe)
+>>> recordings["average_level"].data_frequency
+'15min'
+>>> recordings["pump_state"].recording_mode
+'on_change'
+>>> recordings["pump_state"].data_frequency
+'1h'
+
          +
          +

          + +
          +
          +hdhelpers.metadata.get_series_default_aggregation_method(series)
          +

          Gets the method a Series should be aggregated with by default

          +

          See get_default_aggregation_methods() for the method and its default.

          +
          +
          Parameters:
          +

          series (pd.Series) – Series with metadata following the convention.

          +
          +
          Returns:
          +

          The default aggregation method, e.g. “MEAN”, “SUM” or “LAST”.

          +
          +
          Return type:
          +

          str

          +
          +
          Raises:
          +

          TypeError – If series is not a Series.

          +
          +
          +

          Code example:

          +
          >>> series.attrs = {
+...     "dataset_metadata": {"single_metric": "rainfall"},
+...     "metrics": [{"id": "rainfall", "default_aggregation_method": "SUM"}],
+... }
+>>> get_series_default_aggregation_method(series)
+'SUM'
+>>> series.attrs = {}
+>>> get_series_default_aggregation_method(series)
+'MEAN'
+
          +
          +
          +
          hdhelpers.metadata.get_series_default_timezone(series)
          @@ -555,6 +758,45 @@

          Functions +
          +hdhelpers.metadata.get_series_recording(series)
          +

          Gets how the values of a Series are recorded

          +

          See get_recordings() for the recording and how its frequency falls back to the one of +the dataset metadata.

          +
          +
          Parameters:
          +

          series (pd.Series) – Series with metadata following the convention.

          +
          +
          Returns:
          +

          The recording of the metric, with the frequency of the dataset metadata if the metric does not provide one.

          +
          +
          Return type:
          +

          Recording

          +
          +
          Raises:
          +

          TypeError – If series is not a Series.

          +
          +
          +

          Code example:

          +
          >>> series.attrs = {
+...     "dataset_metadata": {"single_metric": "water.north.level", "metric_key": "external_id"},
+...     "metrics": [
+...         {
+...             "external_id": "water.north.level",
+...             "recording_mode": "periodic",
+...             "data_frequency": "2h",
+...             "data_frequency_timezone": "Europe/Berlin",
+...         }
+...     ],
+... }
+>>> recording = get_series_recording(series)
+>>> recording.data_frequency, recording.data_frequency_offset, recording.data_frequency_timezone
+('2h', None, 'Europe/Berlin')
+
          +
          +

          +
          hdhelpers.metadata.get_series_short_display_name(series, *, include_inherited=False)
          @@ -685,6 +927,41 @@

          Functions +
          +hdhelpers.metadata.get_singlets_default_aggregation_method(singletsframe)
          +

          Gets the method the single metric of a SingleTSFrame should be aggregated with by default

          +

          See get_default_aggregation_methods() for the method and its default.

          +
          +
          Parameters:
          +

          singletsframe (pd.DataFrame) – STSF with metadata following the convention.

          +
          +
          Returns:
          +

          The default aggregation method, e.g. “MEAN”, “SUM” or “LAST”.

          +
          +
          Return type:
          +

          str

          +
          +
          Raises:
          +

          TypeError – If singletsframe is not a DataFrame.

          +
          +
          +

          Code example:

          +
          >>> dataframe.attrs = {
+...     "dataset_metadata": {"single_metric": "valve"},
+...     "metrics": [
+...         {
+...             "id": "valve",
+...             "value_dimensions": [{"column": "value", "value_data_type": "string"}],
+...         }
+...     ],
+... }
+>>> get_singlets_default_aggregation_method(dataframe)
+'LAST'
+
          +
          +

          +
          hdhelpers.metadata.get_singlets_default_timezone(singletsframe)
          @@ -947,6 +1224,42 @@

          Functions +
          +hdhelpers.metadata.get_singlets_recording(singletsframe)
          +

          Gets how the values of the single metric of a SingleTSFrame are recorded

          +

          See get_recordings() for the recording and how its frequency falls back to the one of +the dataset metadata.

          +
          +
          Parameters:
          +

          singletsframe (pd.DataFrame) – STSF with metadata following the convention.

          +
          +
          Returns:
          +

          The recording of the metric, with the frequency of the dataset metadata if the metric does not provide one.

          +
          +
          Return type:
          +

          Recording

          +
          +
          Raises:
          +

          TypeError – If singletsframe is not a DataFrame.

          +
          +
          +

          Code example:

          +
          >>> dataframe.attrs = {
+...     "dataset_metadata": {
+...         "single_metric": "abc.temp",
+...         "ref_data_frequency": "5min",
+...         "ref_data_frequency_offset": "1min",
+...     },
+...     "metrics": [{"id": "abc.temp", "recording_mode": "periodic"}],
+... }
+>>> recording = get_singlets_recording(dataframe)
+>>> recording.recording_mode, recording.data_frequency, recording.data_frequency_offset
+('periodic', '5min', '1min')
+
          +
          +

          +
          hdhelpers.metadata.get_singlets_short_display_names(singletsframe, *, include_inherited=False)
          @@ -1105,6 +1418,39 @@

          Functions +
          +hdhelpers.metadata.without_metadata(data)
          +

          Gets a copy of a Series or DataFrame without its metadata, to compute on

          +

          pandas deep-copies the metadata (attrs) of a Series or DataFrame to every object an +operation creates from it, e.g. when filtering rows, selecting a column or grouping. For +metadata of many metrics, e.g. with comments, loops over the metrics then take much longer +than the computation itself. Compute on the copy without metadata instead and attach the +metadata of the result once at the end.

          +

          The copy is cheap: It shares the data with data until one of them is changed (copy on +write), so changing the copy does not change data.

          +
          +
          Parameters:
          +

          data (pd.Series | pd.DataFrame) – Series or DataFrame, e.g. a MTSF with metadata.

          +
          +
          Returns:
          +

          Copy of data with empty attrs.

          +
          +
          Return type:
          +

          pd.Series | pd.DataFrame

          +
          +
          +

          Code example:

          +
          >>> multitsframe.attrs = {"dataset_metadata": {"metric_key": "id"}, "metrics": []}
+>>> data = without_metadata(multitsframe)
+>>> data.attrs
+{}
+>>> multitsframe.attrs
+{'dataset_metadata': {'metric_key': 'id'}, 'metrics': []}
+
          +
          +

          +

          helpers

          @@ -1320,29 +1666,44 @@

          Table of Contents

        • Further Information
        • Functions
          • metadata
          • helpers
              diff --git a/docs/objects.inv b/docs/objects.inv index 3fa5d63a4fb0da27092f66224ce6944cab09352a..4b30d33fba88edda9e93fd9798332f08d8c50d23 100644 GIT binary patch delta 554 zcmV+_0@eMT1eXPncz>6{ZrmUchVOm~OYL>7=9+U;HHS*8<^d3a!A8a4!Z?ll^a0yA zxY;IN?329%|Hr`mV7B26s1B^F;F&>((r8z9!)`72l1b2bRt8b(`YT1cnKeJJntwuk)__DAM!m?(MX%Y$ z(JGIXVjw76_-piLc?(GSJiE==Mp5+>n6cb1P>?f4o!c_jtE>t?iLmgy2hD~+p@>#K z_6WO3#0cR44+N$Qn~SvZjnKNbP}5Ky;yX5mAH((K%E*u`fJ+rUtq||J;37SEnuDOd z!VB|u*E)5@i+@$gq)68(IuK3UOhOCP^%h!xR^$SFp%{@`m@IvM>fYB@0{25B*e9y8 zAd7MGa&RF9_l`S`pynQlD^}u^J)G=1&-%n~doX*aVbfD07vdUY@vxQcwai?SH)&*g z*)N9zJl@U!a`*Xpp&d8jd{all?qf6|?K@8w&;jeRbAOja%;5$NTO|*Xr~D(m`TOx( zx7;8TAB&8D;FXLstIs=r^!74!xVOf$fLW3c z^+D)d>5$)Y!@{jA36FitZy`{QfuaUIQbvKIkD!7iMyLMw;k9_}Hgl66OWH;F|L~UA s4X4;J2_t??b(qLW>j$#);4PrnY6sM^W-XjzljdP+KhqkD7k+Fn4RU252><{9 delta 430 zcmV;f0a5;!1)Kzscz>3`PJ}QJhVMK@6TL2Rub$QD!DJI3fYf$?#?oSEFyhl&C?K2F z2!&J0wEs6VooP!cOVBkqOyN^NP0AW%b$eS`*=S%lOc@EEX-Sp73bw;FJ0#^Vvh=Yx z-H(P2GtIK&ob)H+$zYS`N+>XN(w0b>;*Jwv%o@g3DU4$Qlz(X}xEFbO-WajaTk5eY zZ!=OAd|9(tUIB4AWVbllQPhVkA?J;z$uq8X9ViNG&D+fIDw0qE$HqbB%N7eDr$nYq zZlm<^p}#la6iDxyz60U2Wa&N zWWs9e*1Am52g?g%k$e@Ygyo0#3?aK?BHsQ{kmRkjf*p=*X71Orht9xy zLsEe0DZN1RSJ0hgw1)Lh-DmQ%Y~apTOkpS855Xm`ZYK}pL=ZfOIt21;_8-})rP619 YrbggSl;^@=3!0ZOf8qrs->X}r(eH23#{d8T diff --git a/docs/searchindex.js b/docs/searchindex.js index 4bca6aa..75b26e4 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"Which time zone to compute in?":[[0,"which-time-zone-to-compute-in"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"get_default_timezones() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_timezones",false]],"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_series_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_timezone",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_timezone",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,2,1,"","get_default_timezones"],[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_series_default_timezone"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_default_timezone"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function"},"terms":{"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"1d":0,"A":[0,1],"As":1,"Both":0,"DO":0,"Each":1,"For":[0,1],"How":1,"If":[0,1],"In":[0,1],"It":[0,1],"NOT":0,"Other":0,"Our":0,"So":0,"That":0,"The":[0,1],"This":1,"To":1,"We":0,"Where":1,"Which":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"accept":[0,1],"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adher":0,"adit":1,"adjust":1,"aggreg":[0,1],"agnost":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"america":[0,1],"ani":[0,1],"anoth":1,"anyway":0,"appli":[0,1],"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attr":1,"avail":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":[0,1],"berlin_sensor":1,"bool":1,"boolean":1,"bucket":[0,1],"by_metr":1,"c":1,"calendar":0,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"class":1,"close":0,"cm":0,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":0,"compli":1,"compon":[0,1],"comput":1,"concret":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"current":1,"custom":1,"daili":0,"daily_sum":0,"dark":1,"dashboard":0,"data":[0,1],"datafram":1,"dataset":[0,1],"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"day":[0,1],"daylight":0,"de":1,"de_d":1,"dedic":[0,1],"def":0,"default":[0,1],"default_timezon":[0,1],"defaultdict":1,"defin":[0,1],"depend":0,"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":[0,1],"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":[0,1],"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":[0,1],"evalu":[0,1],"everi":0,"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":[0,1],"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"final":1,"find":0,"firmwar":0,"first":1,"fix":[0,1],"follow":[0,1],"format":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gaug":0,"generat":1,"german":1,"germani":0,"get":1,"get_default_timezon":[0,1],"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_series_default_timezon":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_default_timezon":[0,1],"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"group":0,"groupbi":0,"handl":[0,1],"happen":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"hour":0,"iana":[0,1],"id":[0,1],"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":[0,1],"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":[0,1],"local":1,"longitud":0,"look":[0,1],"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"may":0,"meaning":1,"meant":[0,1],"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"metric_without_metadata":1,"midnight":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":[0,1],"nest":1,"never":0,"nevertheless":0,"new":0,"new_york":[0,1],"new_york_sensor":1,"next":[0,1],"non":1,"none":[0,1],"north":[0,1],"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":[0,1],"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"output":1,"packag":1,"panda":[0,1],"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":0,"place":1,"plain":0,"plant":[0,1],"platform":[0,1],"pleas":1,"plot_target_set":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pytz":1,"queri":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"red":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"report":0,"repres":0,"request":[0,1],"requir":0,"resampl":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":[0,1],"result":[0,1],"retriev":[0,1],"return":[0,1],"ruhr":1,"ruhrverband":0,"s":[0,1],"save":0,"say":0,"scatter":[0,1],"second":1,"see":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_index":0,"shade":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":[0,1],"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_metadata":1,"singletsfram":1,"skip":0,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"sum":0,"surpress":1,"switch":[0,1],"t":1,"target":[0,1],"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timeseri":[0,1],"timestamp":[0,1],"timezon":[0,1],"to_datetim":1,"to_timezon":[0,1],"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":[0,1],"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"user":0,"usual":0,"utc":[0,1],"utcoffset":1,"valu":[0,1],"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"vari":1,"variabl":1,"via":[0,1],"view":0,"viewer":0,"vision":0,"want":0,"water":[0,1],"way":1,"well":1,"whether":1,"whole":0,"whose":[0,1],"will":1,"win":0,"without":[0,1],"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"york":0,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"Which":0,"come":0,"comput":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"time":0,"use":0,"zone":0}}) \ No newline at end of file +Search.setIndex({"alltitles":{"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Introduction":[[1,"introduction"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"Which time zone to compute in?":[[0,"which-time-zone-to-compute-in"]],"exceptions":[[1,"module-hdhelpers.exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"module-hdhelpers.metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["first_steps.rst","index.rst"],"indexentries":{"data_frequency (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency",false]],"data_frequency_offset (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency_offset",false]],"data_frequency_timezone (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency_timezone",false]],"get_default_aggregation_methods() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_aggregation_methods",false]],"get_default_timezones() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_timezones",false]],"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_recordings() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_recordings",false]],"get_series_default_aggregation_method() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_aggregation_method",false]],"get_series_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_timezone",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_recording() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_recording",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_default_aggregation_method() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_aggregation_method",false]],"get_singlets_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_timezone",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_recording() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_recording",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"recording (class in hdhelpers.metadata)":[[1,"hdhelpers.metadata.Recording",false]],"recording_aggregation_method (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.recording_aggregation_method",false]],"recording_mode (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.recording_mode",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]],"without_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.without_metadata",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.helpers":[[1,2,1,"","get_locale"],[1,2,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,3,1,"","Recording"],[1,2,1,"","get_default_aggregation_methods"],[1,2,1,"","get_default_timezones"],[1,2,1,"","get_display_names"],[1,2,1,"","get_measurements"],[1,2,1,"","get_metric_info"],[1,2,1,"","get_names"],[1,2,1,"","get_queried_interval"],[1,2,1,"","get_recordings"],[1,2,1,"","get_series_default_aggregation_method"],[1,2,1,"","get_series_default_timezone"],[1,2,1,"","get_series_display_name"],[1,2,1,"","get_series_info"],[1,2,1,"","get_series_measurement"],[1,2,1,"","get_series_name"],[1,2,1,"","get_series_recording"],[1,2,1,"","get_series_short_display_name"],[1,2,1,"","get_series_unit"],[1,2,1,"","get_short_display_names"],[1,2,1,"","get_singlets_default_aggregation_method"],[1,2,1,"","get_singlets_default_timezone"],[1,2,1,"","get_singlets_display_names"],[1,2,1,"","get_singlets_info"],[1,2,1,"","get_singlets_measurements"],[1,2,1,"","get_singlets_metric_info"],[1,2,1,"","get_singlets_names"],[1,2,1,"","get_singlets_recording"],[1,2,1,"","get_singlets_short_display_names"],[1,2,1,"","get_singlets_units"],[1,2,1,"","get_units"],[1,2,1,"","without_metadata"]],"hdhelpers.metadata.Recording":[[1,4,1,"","data_frequency"],[1,4,1,"","data_frequency_offset"],[1,4,1,"","data_frequency_timezone"],[1,4,1,"","recording_aggregation_method"],[1,4,1,"","recording_mode"]],"hdhelpers.plotting":[[1,2,1,"","resolve_color"],[1,2,1,"","set_agnostic_theme"],[1,2,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"],"3":["py","class","Python class"],"4":["py","attribute","Python attribute"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function","3":"py:class","4":"py:attribute"},"terms":{"000z":[],"00z":1,"01t01":1,"01t08":0,"05t13":1,"06t13":1,"15min":1,"1d":[0,1],"1h":1,"1min":1,"1ms":1,"2h":1,"5min":1,"A":[0,1],"Both":0,"DO":0,"Each":1,"For":[0,1],"How":1,"If":[0,1],"In":[0,1],"It":[0,1],"NOT":0,"Other":0,"Our":0,"So":0,"That":0,"The":[0,1],"They":1,"This":1,"To":1,"We":0,"Where":1,"Which":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"abc":1,"abov":0,"absent":1,"abstract":1,"accept":[0,1],"access":1,"accord":0,"acm":[0,1],"addit":0,"adequ":1,"adher":0,"adit":1,"adjust":1,"aggreg":[0,1],"agnost":1,"alia":1,"align":1,"all_timezon":1,"allow":[0,1],"also":[0,1],"alway":0,"ambigu":1,"america":[0,1],"ani":[0,1],"anoth":1,"anyway":0,"appli":[0,1],"applic":1,"arbitrari":1,"arbitrarili":1,"arg":1,"argument":0,"asset":[0,1],"associ":1,"assum":1,"attach":1,"attr":1,"avail":1,"average_level":1,"avg":1,"axe":1,"axi":0,"back":[0,1],"background":1,"base":1,"befor":[0,1],"behaviour":1,"belong":[0,1],"berlin":[0,1],"berlin_sensor":1,"bool":1,"boolean":1,"bucket":[0,1],"by_metr":1,"c":1,"calendar":0,"calibr":0,"calibration_d":0,"can":[0,1],"carri":1,"case":1,"caught":0,"certain":1,"chang":1,"cheap":1,"class":1,"close":0,"cm":0,"coarser":1,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1],"column_nam":1,"come":1,"comment":[0,1],"compli":1,"compon":[0,1],"comput":1,"concret":1,"consist":1,"contain":[0,1],"content":0,"context":1,"contrast":[0,1],"control":1,"convent":[0,1],"convert":1,"convert_index":1,"copi":1,"correspond":[0,1],"count":[0,1],"counterpart":1,"creat":1,"current":1,"custom":1,"daili":0,"daily_sum":0,"dark":1,"dashboard":0,"data":[0,1],"data_frequ":1,"data_frequency_offset":1,"data_frequency_timezon":1,"datafram":1,"dataset":[0,1],"dataset_metadata":[0,1],"date":[0,1],"datetim":1,"day":[0,1],"daylight":0,"de":1,"de_d":1,"deadband":1,"dedic":[0,1],"deep":1,"def":0,"default":[0,1],"default_aggregation_method":1,"default_timezon":[0,1],"default_titl":[],"defaultdict":1,"defin":[0,1],"depend":0,"describ":0,"design":[0,1],"determin":1,"dict":1,"dictionari":[0,1],"differ":0,"dimens":[0,1],"direct":[0,1],"display":[0,1],"display_nam":1,"display_name_of_metric1":1,"display_name_of_seri":1,"doe":1,"dot":[0,1],"e":[0,1],"ea1255":1,"eas":1,"eb6962":1,"edit":0,"either":0,"empti":[0,1],"en":1,"enabl":1,"end":[0,1],"energi":1,"entri":[0,1],"entrypoint":0,"equival":1,"error":[0,1],"error_cod":1,"especi":1,"europ":[0,1],"evalu":[0,1],"everi":[0,1],"exact":1,"exampl":[0,1],"exclus":1,"execut":1,"expect":1,"explicit":[0,1],"explicit_local":1,"external_first":1,"external_id":[0,1],"external_second":1,"extra_inform":1,"extract":1,"fall":[0,1],"fallback":0,"fals":[0,1],"field":[0,1],"fig":[0,1],"figur":[0,1],"filter":1,"final":1,"find":0,"firmwar":0,"first":1,"fix":[0,1],"follow":[0,1],"format":1,"frequenc":1,"frontend":1,"full_titl":0,"function":0,"fuseki":1,"g":[0,1],"gap":1,"gaug":0,"generat":1,"german":1,"germani":0,"get":1,"get_default_aggregation_method":1,"get_default_timezon":[0,1],"get_display_nam":1,"get_local":[0,1],"get_measur":1,"get_metric_info":[0,1],"get_nam":1,"get_queried_interv":[0,1],"get_record":1,"get_series_default_aggregation_method":1,"get_series_default_timezon":[0,1],"get_series_display_nam":1,"get_series_info":[0,1],"get_series_measur":1,"get_series_nam":[0,1],"get_series_record":1,"get_series_short_display_nam":1,"get_series_unit":[0,1],"get_short_display_nam":1,"get_singlets_default_aggregation_method":1,"get_singlets_default_timezon":[0,1],"get_singlets_display_nam":1,"get_singlets_info":[0,1],"get_singlets_measur":1,"get_singlets_metric_info":[0,1],"get_singlets_nam":1,"get_singlets_record":1,"get_singlets_short_display_nam":1,"get_singlets_unit":[0,1],"get_unit":[0,1],"get_value_dimension_info":1,"given":[0,1],"global":1,"glom":[0,1],"go":[0,1],"good":1,"graph_object":[0,1],"group":[0,1],"groupbi":0,"handl":[0,1],"happen":1,"heartbeat":1,"helper":0,"helperexcept":1,"hetida":[0,1],"hierarchi":[0,1],"higher":[0,1],"hold":1,"hour":0,"iana":[0,1],"id":[0,1],"identifi":1,"import":[0,1],"includ":1,"include_inherit":[0,1],"index":[0,1],"infer":1,"info":1,"inherit":[0,1],"inspir":1,"instal":1,"instead":[0,1],"insuffici":1,"insufficientplottingdata":1,"int":1,"integr":1,"interpret":[0,1],"interv":[0,1],"irregular":1,"iso":1,"isoformat":1,"json":[0,1],"just":1,"key":[0,1],"keyword":0,"ki":[0,1],"kwarg":1,"label":[0,1],"languag":1,"last":[0,1],"latest":1,"latitud":[0,1],"latter":1,"let":[0,1],"level":[0,1],"light":1,"like":[0,1],"limit":0,"line":0,"list":[0,1],"local":1,"longer":1,"longitud":0,"look":[0,1],"loop":1,"m":1,"main":0,"mainten":0,"mani":1,"manufactur":[0,1],"marker":0,"max":1,"may":0,"mean":1,"meaning":1,"meant":[0,1],"measur":[0,1],"messag":1,"method":1,"metric":[0,1],"metric1":1,"metric2":1,"metric3":1,"metric_info":1,"metric_key":[0,1],"metric_without_metadata":1,"midnight":[0,1],"might":0,"miss":[0,1],"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"mtsf":1,"much":1,"multipl":1,"multitsfram":[0,1],"name":[0,1],"name_of_metric1":1,"name_of_metric2":1,"name_of_series_1":1,"name_of_series_2":1,"neither":[0,1],"nest":1,"never":0,"nevertheless":0,"new":0,"new_york":[0,1],"new_york_sensor":1,"next":[0,1],"non":1,"none":[0,1],"normal":1,"north":[0,1],"note":[0,1],"null":[0,1],"object":[0,1],"object_to_convert":1,"offset":[0,1],"on_chang":1,"onc":1,"one":[0,1],"onli":[0,1],"oper":[0,1],"option":1,"orang":1,"order":0,"otherwis":1,"output":1,"packag":1,"panda":[0,1],"paramet":1,"pass":0,"path":[0,1],"pd":1,"per":[0,1],"period":1,"place":1,"plain":0,"plant":[0,1],"platform":[0,1],"pleas":1,"plot_target_set":1,"point":1,"possibl":1,"present":[0,1],"print":1,"properti":[0,1],"provid":[0,1],"provis":0,"pump_run":1,"pump_stat":1,"pytz":1,"queri":1,"rainfal":1,"rais":[0,1],"rang":0,"rather":0,"re":1,"reach":[0,1],"receiv":0,"record":1,"recording_aggregation_method":1,"recording_mod":1,"red":1,"ref_data_frequ":1,"ref_data_frequency_offset":1,"ref_data_frequency_timezon":1,"ref_interval_end_timestamp":[0,1],"ref_interval_start_timestamp":[0,1],"ref_interval_typ":0,"replac":0,"report":[0,1],"repres":0,"request":[0,1],"requir":0,"resampl":0,"resolv":1,"resolve_color":[0,1],"resort":[0,1],"respect":[0,1],"result":[0,1],"retriev":[0,1],"return":[0,1],"row":1,"ruhr":1,"ruhrverband":0,"s":[0,1],"sampl":1,"save":0,"say":0,"scatter":[0,1],"scheme":[],"second":1,"see":1,"select":1,"sensibl":1,"sensor":[0,1],"separ":0,"seri":[0,1],"serial":1,"set":[0,1],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_index":0,"shade":1,"share":1,"shift":1,"short":[0,1],"short_display_nam":[0,1],"short_display_name_of_metric1":1,"short_display_name_of_seri":1,"show":[0,1],"similar":1,"sinc":[0,1],"singl":1,"single_metr":[0,1],"single_metric_dataset_metadata":[],"single_metric_metadata":1,"singletsfram":1,"skip":0,"spec":[0,1],"specifi":1,"split":0,"standard":[0,1],"start":[0,1],"state":1,"step":1,"str":[0,1],"streamlin":1,"string":[0,1],"structured_metadata":1,"stsf":1,"style":1,"sub":[0,1],"sum":[0,1],"surpress":1,"switch":[0,1],"t":1,"take":1,"taken":1,"target":[0,1],"tbd":1,"temp":1,"temperatur":1,"templat":1,"test":1,"theme":1,"therefor":1,"tick":1,"time":1,"timedelta":1,"timeseri":[0,1],"timestamp":[0,1],"timezon":[0,1],"to_datetim":1,"to_timezon":[0,1],"togeth":1,"toggleabl":1,"total_second":1,"transform":[0,1],"tri":1,"true":[0,1],"tupl":1,"type":1,"typeerror":1,"tz_convert":[0,1],"underlying":1,"unit":[0,1],"unknown":1,"update_layout":0,"update_xax":0,"upper":0,"us":1,"use":1,"user":0,"usual":0,"utc":[0,1],"utcoffset":1,"valu":[0,1],"value_data_typ":1,"value_dim_1":1,"value_dim_info":1,"value_dimens":[0,1],"value_dimensions_shar":[0,1],"value_name_of_seri":1,"valueerror":1,"valv":1,"vari":1,"variabl":1,"via":[0,1],"view":0,"viewer":0,"vision":0,"want":0,"water":[0,1],"way":1,"well":1,"whether":1,"whole":[0,1],"whose":[0,1],"will":1,"win":0,"without":[0,1],"without_metadata":1,"write":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"york":0,"zone":1,"zoom":1},"titles":["First steps","hdhelpers"],"titleterms":{"Further":1,"How":0,"Where":0,"Which":0,"come":0,"comput":0,"doe":0,"except":1,"first":0,"function":1,"get":0,"hdhelper":[0,1],"helper":1,"inform":[0,1],"introduct":1,"metadata":[0,1],"nest":0,"plot":[0,1],"queri":0,"step":0,"tbd":0,"time":0,"use":0,"zone":0}}) \ No newline at end of file From b7494a72809a2644efc19c0e50cfd37aaa6fb967 Mon Sep 17 00:00:00 2001 From: jenny kupzig Date: Tue, 27 Oct 2026 08:49:53 +0100 Subject: [PATCH 14/15] formatting --- src/hdhelpers/metadata/__init__.py | 10 +++++----- src/hdhelpers/metadata/helpers.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/hdhelpers/metadata/__init__.py b/src/hdhelpers/metadata/__init__.py index 969d034..e068e2e 100644 --- a/src/hdhelpers/metadata/__init__.py +++ b/src/hdhelpers/metadata/__init__.py @@ -5,31 +5,31 @@ from hdhelpers.metadata.helpers import ( Recording, get_default_aggregation_methods, - get_recordings, - get_series_default_aggregation_method, - get_series_recording, - get_singlets_default_aggregation_method, - get_singlets_recording, get_default_timezones, get_display_names, get_measurements, get_metric_info, get_names, get_queried_interval, + get_recordings, + get_series_default_aggregation_method, get_series_default_timezone, get_series_display_name, get_series_info, get_series_measurement, get_series_name, + get_series_recording, get_series_short_display_name, get_series_unit, get_short_display_names, + get_singlets_default_aggregation_method, get_singlets_default_timezone, get_singlets_display_names, get_singlets_info, get_singlets_measurements, get_singlets_metric_info, get_singlets_names, + get_singlets_recording, get_singlets_short_display_names, get_singlets_units, get_units, diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 0d68e19..a225895 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -17,13 +17,6 @@ from glom import Coalesce, Spec, glom import hdhelpers.metadata._specs as _specs -from hdhelpers.metadata._recording import ( - Recording, - default_aggregation_method_by_metric, - recording_by_metric, - single_metric_default_aggregation_method, - single_metric_recording, -) from hdhelpers.metadata._helpers import ( check_dataframe, check_series, @@ -33,13 +26,20 @@ get_value_dimension_info, metric_info_by_metric, series_value_info, + single_metric_default_timezone, singlets_metric_info, singlets_value_dimension_info, - single_metric_default_timezone, singlets_value_dimension_info_with_fallbacks, spec_not_none, value_dimension_info_with_fallbacks, ) +from hdhelpers.metadata._recording import ( + Recording, + default_aggregation_method_by_metric, + recording_by_metric, + single_metric_default_aggregation_method, + single_metric_recording, +) def get_units( From 79c0fd00688e38a512e40d37003c92e0b84fe93c Mon Sep 17 00:00:00 2001 From: jenny kupzig Date: Tue, 27 Oct 2026 08:52:01 +0100 Subject: [PATCH 15/15] update python --- .github/workflows/check_pull_request.yml | 4 + CHANGELOG.md | 1 + README.md | 9 +- pyproject.toml | 12 +- run | 13 +- src/hdhelpers/__init__.py | 2 +- uv.lock | 198 +++++++++++++++++++++-- 7 files changed, 207 insertions(+), 32 deletions(-) diff --git a/.github/workflows/check_pull_request.yml b/.github/workflows/check_pull_request.yml index 10979ee..a2b716d 100644 --- a/.github/workflows/check_pull_request.yml +++ b/.github/workflows/check_pull_request.yml @@ -23,6 +23,7 @@ jobs: - "3.12" - "3.13" - "3.14" + - "3.15" steps: - uses: actions/checkout@v6 @@ -30,6 +31,7 @@ jobs: - name: Install uv and set the Python version uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: + version: ">=0.13.4" python-version: ${{ matrix.python-version }} - name: Display Python version @@ -51,6 +53,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: ">=0.13.4" - name: Set up Python run: uv python install diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f6cc58..635658d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## [Unreleased] +- Update python versions to fix CA certificate issue (https://emudhra.we-have-a-problem.com/) - metadata accessors can fall back to the properties a metric inherits from objects higher up in an asset hierarchy (under "inherited" in the metric metadata of the current convention), only as last resort after "value_dimensions_shared". This is the default for get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info. The functions for dedicated information (units, names, display names, short display names and measurements) only do so with keyword argument include_inherited=True, since an inherited property with the same label usually does not describe the metric itself - get_metric_info and get_singlets_metric_info fall back to the value dimension "value" and to "value_dimensions_shared" if the metric itself does not provide the information - fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension diff --git a/README.md b/README.md index 83de197..ec783be 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/hetida/hdhelpers/graphs/commit-activity) [![build](https://github.com/hetida/hdhelpers/actions/workflows/check_pull_request.yml/badge.svg)](https://github.com/hetida/hdhelpers/actions/workflows/check_pull_request.yml) -![python](https://img.shields.io/badge/python-%203.12%20|%203.13|%203.14%20-blue) +![python](https://img.shields.io/badge/python-%203.12%20|%203.13|%203.14%20|%203.15%20-blue) [![PyPI version](https://badge.fury.io/py/hdhelpers.svg)](https://badge.fury.io/py/hdhelpers) ![PyPI - License](https://img.shields.io/pypi/l/hdhelpers) [![Downloads](https://pepy.tech/badge/hdhelpers)](https://pepy.tech/project/hdhelpers) @@ -100,12 +100,7 @@ To **publish** the build from the `dist` subdirectory to PyPI, 3) verifiy that package can be installed by using: ```bash -uv run --with 'hdhelpers==' - --refresh-package hdhelpers - --default-index https://test.pypi.org/simple/ - --index https://test.pypi.org/simple/ - --no-project - -- python -c "import hdhelpers; print(hdhelpers.__version__)" +uv run --with 'hdhelpers==0.0.9' --refresh-package hdhelpers --default-index https://test.pypi.org/simple/ --index https://test.pypi.org/simple/ --no-project -- python -c "import hdhelpers; print(hdhelpers.__version__)" ``` 4) use `uv publish dist/*` to upload the package on [hdhelpers PyPI project](https://pypi.org/project/hdhelpers/). diff --git a/pyproject.toml b/pyproject.toml index f9f52e0..c9cef9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ exclude = [ [project] name = "hdhelpers" -version = "0.0.9" +version = "0.0.10" description = "Streamlines metadata & timezone handling, and plotting in hetida designer components" readme = "README.md" maintainers = [ @@ -29,11 +29,13 @@ maintainers = [ requires-python = ">=3.12" dependencies = [ - "pandas>=3", + "pandas>=3; python_version < '3.15'", + "pandas>=3.0.6; python_version >= '3.15'", "plotly", "pydantic", "glom>25,<26", - "numpy", + "numpy; python_version < '3.15'", + "numpy>=2.5.2; python_version >= '3.15'", "pytz", ] license = "MIT" @@ -52,6 +54,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "License :: OSI Approved :: MIT License", ] @@ -88,9 +91,10 @@ lint.ignore = ["E713", "E714", "B008", "PLR2004", "COM812", "RET504", "PLR0913", [dependency-groups] dev = [ + "certifi>=2026.10.19", "mypy>=1.19.1", "pandas-stubs>=3.0.0.260204", - "pip>=26.0.1", + "pip>=26.2.2", "pytest>=9.0.2", "pytest-cov>=7.1.0", "ruff>=0.15.7", diff --git a/run b/run index 344e5ff..5622cdd 100755 --- a/run +++ b/run @@ -110,17 +110,20 @@ install_editable_package(){ } build_package(){ + local package_version uv sync --frozen rm -f -r dist - uv version $@ - sed -i "s/^__version__ = .*/__version__ = \"$@\"/" src/hdhelpers/__init__.py + uv version "$@" + package_version="$(uv version --short)" + sed -i "s/^__version__ = .*/__version__ = \"$package_version\"/" src/hdhelpers/__init__.py uv build --no-sources } test-py-versions() { - echo "Py 3.12" && uv run --python=3.12 --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 && - echo "Py 3.13" && uv run --python=3.13 --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 && - echo "Py 3.14" && uv run --python=3.14 --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 + echo "Py 3.12" && uv run --python '>=3.12.16,<3.13' --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 && + echo "Py 3.13" && uv run --python '>=3.13.18,<3.14' --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 && + echo "Py 3.14" && uv run --python '>=3.14.10,<3.15' --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 && + echo "Py 3.15" && uv run --python '>=3.15.2,<3.16' --all-extras pytest tests -c pytest.ini --cov=. --cov-report=term-missing --cov-fail-under=80 } #----- Execution -----# diff --git a/src/hdhelpers/__init__.py b/src/hdhelpers/__init__.py index 40f1073..a6fa5e3 100644 --- a/src/hdhelpers/__init__.py +++ b/src/hdhelpers/__init__.py @@ -10,7 +10,7 @@ from hdhelpers.plotting.timestamp_ticks import TICKFORMATSTOPS, set_dt_ticks # do not edit line of __version__ as it is automatically modified by running ./run build_package -__version__ = "0.0.9" +__version__ = "0.0.10" # import can be done by using `from hdhelpers import *`` __all__ = [ diff --git a/uv.lock b/uv.lock index 815ed9d..36d12ed 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -60,11 +60,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.4.22" +version = "2026.10.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, ] [[package]] @@ -270,12 +270,14 @@ wheels = [ [[package]] name = "hdhelpers" -version = "0.0.9" +version = "0.0.10" source = { editable = "." } dependencies = [ { name = "glom" }, - { name = "numpy" }, - { name = "pandas" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, + { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "pandas", version = "3.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, { name = "plotly" }, { name = "pydantic" }, { name = "pytz" }, @@ -283,6 +285,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "certifi" }, { name = "mypy" }, { name = "pandas-stubs" }, { name = "pip" }, @@ -296,8 +299,10 @@ dev = [ [package.metadata] requires-dist = [ { name = "glom", specifier = ">25,<26" }, - { name = "numpy" }, - { name = "pandas", specifier = ">=3" }, + { name = "numpy", marker = "python_full_version < '3.15'" }, + { name = "numpy", marker = "python_full_version >= '3.15'", specifier = ">=2.5.2" }, + { name = "pandas", marker = "python_full_version < '3.15'", specifier = ">=3" }, + { name = "pandas", marker = "python_full_version >= '3.15'", specifier = ">=3.0.6" }, { name = "plotly" }, { name = "pydantic" }, { name = "pytz" }, @@ -305,9 +310,10 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "certifi", specifier = ">=2026.10.19" }, { name = "mypy", specifier = ">=1.19.1" }, { name = "pandas-stubs", specifier = ">=3.0.0.260204" }, - { name = "pip", specifier = ">=26.0.1" }, + { name = "pip", specifier = ">=26.2.2" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "ruff", specifier = ">=0.15.7" }, @@ -542,6 +548,14 @@ wheels = [ name = "numpy" version = "2.4.4" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/28/05/32396bec30fb2263770ee910142f49c1476d08e8ad41abf8403806b520ce/numpy-2.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b", size = 16689272, upload-time = "2026-03-29T13:18:49.223Z" }, @@ -599,6 +613,84 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/78/548fb8e07b1a341746bfbecb32f2c268470f45fa028aacdbd10d9bc73aab/numpy-2.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", size = 10566643, upload-time = "2026-03-29T13:21:34.339Z" }, ] +[[package]] +name = "numpy" +version = "2.5.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz", hash = "sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", size = 20866315, upload-time = "2026-10-10T20:05:31.422Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/97/ba2074e92b7befea137e77ea8471e768bbd87c339b7e8c9f5a931949f977/numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c6342f54c67093cae5c0227eb0eb772fdb79f2a2c37a6eb278b9909ee06aa356", size = 17001609, upload-time = "2026-10-10T20:02:40.843Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a9/bac826765e971d8e16e2064e9ac7525fd69b40ac17c905033a7f5442023f/numpy-2.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b11e8fda06a7d69f15ebf542660b74466c2e51094800c1fb794f47ad4faeef17", size = 12015718, upload-time = "2026-10-10T20:02:43.45Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/5ea3570fcb8ccd0882bea99436a513b2c85dad8f774a2057849130a8fb99/numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9cb18a327b49c5c337f972b03682f6a49855525faaf3c0d3e9c96cd0fd8880a8", size = 5451717, upload-time = "2026-10-10T20:02:46.169Z" }, + { url = "https://files.pythonhosted.org/packages/34/f2/b4fc1bafca03868220b5eaf729d2f21ebd7d7b151c0f9e144fe212bbca35/numpy-2.5.4-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:aec3fc4b32ff82421274f5d205c559c51c840c8df66a78efd7f3612dd005a26a", size = 6789926, upload-time = "2026-10-10T20:02:48.139Z" }, + { url = "https://files.pythonhosted.org/packages/dc/96/8319e2457ae4333c62c815c7006b869a4f60985c1e01024c2f8c6c040fe5/numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4d21ab149f15e4e6043dfb0de87e6e5f34ac176cde83060e9802981fca2ac2", size = 15695312, upload-time = "2026-10-10T20:02:50.115Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c799c62e19c337e6d3770b08e475887fb30ce8477d3c09efca6b2f0228a6/numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbde6962867ee75b48b0ee29b2b9372ec5d617799dbaf38e82dc0596f2f7738a", size = 16727283, upload-time = "2026-10-10T20:02:53.186Z" }, + { url = "https://files.pythonhosted.org/packages/39/6b/3604e53fb00314d0dc1b94ec9125a1484f649c0a17480b1f0f0c7a9d6250/numpy-2.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:381a7a3d2e65e64c0ec302795ab9dc12bb1e73f150904699c153716177eebdaf", size = 17047890, upload-time = "2026-10-10T20:02:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7a/e8b58a5289a0d464c52885de47c35a935cdd70c03a4c3ab94a5126416dd0/numpy-2.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b89d0aaae2fe498c648f4c4795c084db535af5bd98ef942b2a3681fb74ce8645", size = 18485839, upload-time = "2026-10-10T20:02:59.018Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c9/47094f597015009f310b8c900def59065ef1ff5a6fe7b51fc65ec58ec2c6/numpy-2.5.4-cp312-cp312-win32.whl", hash = "sha256:9968ab7e49b93ac6e1c3b2239732183152c9150f16308d30b66a372cffe3483c", size = 6138936, upload-time = "2026-10-10T20:03:01.626Z" }, + { url = "https://files.pythonhosted.org/packages/12/33/fefe62073dc8acfd0f2b9ed7c003af2f50aa61555e113e6db02b8f79f145/numpy-2.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7b1b6353e36a7e50de2973a38d705c88ee93adcf120673cee7f45a4a3fa223a", size = 12573091, upload-time = "2026-10-10T20:03:04.349Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/161270b0c2eec56e4c905f6d6d22e1b836887b2cb189d3f5820aa588e9dd/numpy-2.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:aa1cce2ff3f8d953de38b76bf44602caeb69f101430208f64a10067f7cb4b1d3", size = 10521630, upload-time = "2026-10-10T20:03:06.767Z" }, + { url = "https://files.pythonhosted.org/packages/67/14/1c3ee0118a8fce08565a5d8482631608426a33af10a01077fada5dc7c119/numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2377da2dd3ba2c1200956acbab2a358c83b8e1f8531191672d1cd6ad83250d53", size = 16997729, upload-time = "2026-10-10T20:03:09.291Z" }, + { url = "https://files.pythonhosted.org/packages/83/8c/b0ea9477fb1f0d4484bbc5cba21678cc9969704d8d7f3f158d1db35f8e14/numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7415db95818b39ec475a5eea54d9e3b6bc83e3912158e46da3438cdce399804d", size = 12009826, upload-time = "2026-10-10T20:03:11.946Z" }, + { url = "https://files.pythonhosted.org/packages/e2/84/6a3d75b3ba3dfe84ac0053450753d1e6d250a8bf80f66474cc46d1fb643f/numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6d6a71b9d9a97c03633aa12565ef2825ffa036cc1d99cfd50dacf0f128af4fe2", size = 5445803, upload-time = "2026-10-10T20:03:14.329Z" }, + { url = "https://files.pythonhosted.org/packages/61/18/bb993f267ca20b376e07092a16793a5b31ed3138751e9ba480011a14d742/numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d8200f16437b289a5bb927c6e184eccc3e8389bc0070fea4cd5b9e13c1757959", size = 6786220, upload-time = "2026-10-10T20:03:16.602Z" }, + { url = "https://files.pythonhosted.org/packages/db/b6/135bb0953b61dc21c6cafa14b424ae666944e4899cf140e00c2b322a1a45/numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2e71b04c6cad90026e544501bbe0ab9290fa8a4d845e7e8c0d124fb429c988", size = 15689178, upload-time = "2026-10-10T20:03:18.721Z" }, + { url = "https://files.pythonhosted.org/packages/da/24/3bd070f3269dc609d8f26b2643f62ef91bb415841c0b294805aaf7fe06da/numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0", size = 16718044, upload-time = "2026-10-10T20:03:21.386Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8e/9d15bd356b0a019c965312b1a3c6a727cac4cae5bc40045fbc12ce4cff9c/numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2fa3328f784fc8277fc48026f6cad516f5c561c5d8e2e39b3c9e0c8f23223b34", size = 17048364, upload-time = "2026-10-10T20:03:24.468Z" }, + { url = "https://files.pythonhosted.org/packages/dc/fe/9d5b560db964f15871885f2250795d15945f8699e17ef90c0c2ff4c875b2/numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b", size = 18474904, upload-time = "2026-10-10T20:03:27.895Z" }, + { url = "https://files.pythonhosted.org/packages/e9/98/d27552990f1bd611ef3e7466adadc78312ea2df63b83aad47fdc3d3ca8df/numpy-2.5.4-cp313-cp313-win32.whl", hash = "sha256:5258bc06526964be5face2fc6f756857a3f24f21ec3e72ca131337a75b165d6c", size = 6134537, upload-time = "2026-10-10T20:03:30.511Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/140a40398a66b4471211be1affdb6ed24c486d581bd28d07b7f2fcb69540/numpy-2.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:8b4d2fd2d34e5f8c9235ee787de5631a37a28402b15cb80814df973d2be54129", size = 12566113, upload-time = "2026-10-10T20:03:32.612Z" }, + { url = "https://files.pythonhosted.org/packages/34/52/01d205e5e8ccb27b2b0b141e801f22b830198c979111b0fa44771438d9a9/numpy-2.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:bc39ac66a7a9a3fbd6134fda43136b60ffde99c8f4501e64e0d2b24da137babf", size = 10519523, upload-time = "2026-10-10T20:03:35.163Z" }, + { url = "https://files.pythonhosted.org/packages/99/ba/005cb5edd580d2f84d7ca3206b92dc17d4388e56e6f87ffe8f2762f83139/numpy-2.5.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c668b2f0d651605b58892644b0e302c7157f7159544227758c896982ef384b18", size = 17005499, upload-time = "2026-10-10T20:03:37.961Z" }, + { url = "https://files.pythonhosted.org/packages/f3/49/fee7587c33ee35f7977f9051d7f2023d4e7246d62710c80f20c2361ea232/numpy-2.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ffa6ce09a1c6a08e9667dd9c97aa0b14184e8d18f2a14b78b2a2328c9147f076", size = 12019666, upload-time = "2026-10-10T20:03:40.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b2/c6ce165acffceb15a82c07b9cc77d391f86b3f379ba62911908ae5d34b91/numpy-2.5.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:956555e0603a4d38019ae6925711cb9dc43195c076a928accf7ea5d50bddfe53", size = 5455617, upload-time = "2026-10-10T20:03:43.138Z" }, + { url = "https://files.pythonhosted.org/packages/77/7f/dd85ce260a669a89be06842cf355d7353a33e6cfbc590fb8ebb947d88dc9/numpy-2.5.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:2c2c4afffdeb7920e445028dd71eb932cac3e704792e964bc2a232426d4f1255", size = 6791932, upload-time = "2026-10-10T20:03:44.874Z" }, + { url = "https://files.pythonhosted.org/packages/63/d6/34b0a2b0741386a63025a65a2c09caaaaaad6d0ca95b66cd65c30dd7fcb5/numpy-2.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4054173604cd8658796053f1f3bc0befb68ec1c0762c57fdad61e199256a8617", size = 15710899, upload-time = "2026-10-10T20:03:46.839Z" }, + { url = "https://files.pythonhosted.org/packages/16/d5/928078d2b28f26829b138b4a6c3980045022fb409f570657a224ae60ef4e/numpy-2.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d549420b8858885cea8838a727842249218b9c1da24dd517e25c9c7a948310a3", size = 16721710, upload-time = "2026-10-10T20:03:49.489Z" }, + { url = "https://files.pythonhosted.org/packages/f9/cf/673fd1b8f4cd78eb6320e87ec4c90ac19c095644259e3749853a405c70f4/numpy-2.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:823874a507a84af050493b622affde94b6f7c3a0dc22cb2801381bc03b871c00", size = 17066182, upload-time = "2026-10-10T20:03:52.25Z" }, + { url = "https://files.pythonhosted.org/packages/f3/92/a77b5061b1b3e2643928c37976d79ee173e1b171ed158b7a3c61056b41bc/numpy-2.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4e263278bfb5ee6409db8aedbc4cc32973b1b82bc1e8d3c668551d04d83a7e37", size = 18480315, upload-time = "2026-10-10T20:03:55.39Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1d/1486ef3d3fb2279fd93c4c43c1bbbf1ca389a19816696684409f71babaab/numpy-2.5.4-cp314-cp314-win32.whl", hash = "sha256:cfd73180400042a7c532d30c5e287bdd03c59ff9ee1b4c0316af0539e29dfe23", size = 6185739, upload-time = "2026-10-10T20:03:58.186Z" }, + { url = "https://files.pythonhosted.org/packages/52/9a/e1e512ebc948d5b9dd33b08736760f0ebbed2848fd4eda1f553088a6dcee/numpy-2.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:2ca144f15135b6212a5c47b1e2aeca6e412f102f95a2d5d88d8aec77eb255de3", size = 12703552, upload-time = "2026-10-10T20:04:00.28Z" }, + { url = "https://files.pythonhosted.org/packages/2c/05/de709a982d7bbcd688a3fad71f002e9ff80c2db39e03ee726609b610f1d1/numpy-2.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:468397ba3c64427474706e5c9123fe266395496714dc684294eac75cd4930d1e", size = 10803901, upload-time = "2026-10-10T20:04:02.659Z" }, + { url = "https://files.pythonhosted.org/packages/13/34/083570ada3bb2a30fbe5d77c8c6fef9141144a15d33e6f793a67e9749ab8/numpy-2.5.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1ef3aa6d7e29bb13677323114280b05acc57607fa2300e66432d665d5418a162", size = 12138695, upload-time = "2026-10-10T20:04:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/94/06/1f9c24db48eef0c2d1207e3b11fffb0478e39dfd8c1e1be7476936885eed/numpy-2.5.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:98b053943e5a0474ec0da309d2cb9d3f18ea57f8a2067c2ab7b5f763d1068380", size = 5574615, upload-time = "2026-10-10T20:04:07.316Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/593fba2e1560e949123bc7d2fc48b5893d56e58cd4bd5a273d2fbf60b220/numpy-2.5.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:b64a85f40e154983960a4167d4c1d57a50c7f109b3d3264a3a984154e90a8454", size = 6889383, upload-time = "2026-10-10T20:04:09.918Z" }, + { url = "https://files.pythonhosted.org/packages/eb/9f/b799dfdce4e05e80ed4bc815c71ff343a11533b2c0ffc221cae8538cda63/numpy-2.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a813ed7719bf45463c51779e6a98d0385fe905e48447526938a4b8337333d551", size = 15753763, upload-time = "2026-10-10T20:04:12.278Z" }, + { url = "https://files.pythonhosted.org/packages/34/88/16c5f12f86f5ad2817c4d103205131fc6c8acb3d1878af05a1a4f23ec859/numpy-2.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9b80cdf5cedba0e90d93fa5f9a333c4d65bd545cd669b71bb97ce2b703c9d73", size = 16757212, upload-time = "2026-10-10T20:04:14.799Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4f/a1fe40e18a898e6a5089f4f0d891f0a493eb0574d5b34458f0fbe5aa3e5c/numpy-2.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2199ed071f460487c8db2c0e5c0b564494190edb4772fe80f9aad88b2604def5", size = 17116471, upload-time = "2026-10-10T20:04:17.58Z" }, + { url = "https://files.pythonhosted.org/packages/aa/46/e923a11c78e65c1722e7aaad817c06bd591324174b9d28ce5d31eee4d432/numpy-2.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:64f9c9878c1938476365e11ccfb6b770f3b9e5f045ccddc514235041e6959365", size = 18524063, upload-time = "2026-10-10T20:04:20.365Z" }, + { url = "https://files.pythonhosted.org/packages/5a/fa/84ab064514440c1f64a1b21088f2c82756defdd05e07c75ab233899565b2/numpy-2.5.4-cp314-cp314t-win32.whl", hash = "sha256:64d1c8ac28a4077cf987e0a71a7a0ef7e2df70722f07f0baa42dbb7eb6938647", size = 6340926, upload-time = "2026-10-10T20:04:22.865Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7e/6cd886876f435b10685db9b9f7eeb70356f99e052116f4e5f11c5792c714/numpy-2.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:067374eb538c34c745436365cf7b0112595c1d326f21ce4ff340f61230239fbb", size = 12901584, upload-time = "2026-10-10T20:04:24.99Z" }, + { url = "https://files.pythonhosted.org/packages/38/1b/3c1684f6a06f7307f2335fca6e486cb162847fb97e91d65f8eb5cabad213/numpy-2.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:e94aef2c639da4a960ad0db8e06471208d8589974953d78b61d345b4eb99e394", size = 10891152, upload-time = "2026-10-10T20:04:27.52Z" }, + { url = "https://files.pythonhosted.org/packages/08/f4/3224deff3af2bef6bc0b175369698d8cb348f3d91d9bb0286cd5c9eae9e0/numpy-2.5.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:8dddfbee2e68d26d0d7d7d9cb247b1fd4409241cce32d815a11d97ec2cfde179", size = 17003231, upload-time = "2026-10-10T20:04:30.021Z" }, + { url = "https://files.pythonhosted.org/packages/be/75/fee0b8c6d94b44b2fdfae74f6a4ad5a138739589a8aebaec28ce4e713ed5/numpy-2.5.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:81e3420b27048b65eb14c3acf0c174a8cb0e023277716110347d2dcb26026dad", size = 12018300, upload-time = "2026-10-10T20:04:32.519Z" }, + { url = "https://files.pythonhosted.org/packages/47/c0/d0b335a499a04b65f532c3f034346ef390f81299060f928492dabc1e0272/numpy-2.5.4-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:0b4724a19de67bea8cfc4970798efa78bcbbe2ac2613cfac16721a42d44de2a5", size = 5454250, upload-time = "2026-10-10T20:04:34.943Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0e/461b3783c03d668052e6a21b01b673db6ffcb7831fd32d9aa5368c1cd426/numpy-2.5.4-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:2132418bf8dd124a427ca9e6a1daf9ee1a87185344c95119ceae868b99466da1", size = 6789644, upload-time = "2026-10-10T20:04:37.258Z" }, + { url = "https://files.pythonhosted.org/packages/b3/02/5dad269b02166965a7b4ca14adaddd75dbee0de42435bfecf561b84ba5a6/numpy-2.5.4-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:325518d4245b9e331387702aa58c2ce1dc4cdcbb41dfb4ccd5dcbc7e08db1266", size = 15704353, upload-time = "2026-10-10T20:04:39.616Z" }, + { url = "https://files.pythonhosted.org/packages/93/3a/01360c8036822ed9f7aa32189a77d1476567ec1e8e1383522389e4faac45/numpy-2.5.4-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56733449d2544178beaa4545cee357370440cf056c197f9c7bfb19dbfdd0e86d", size = 16718648, upload-time = "2026-10-10T20:04:42.383Z" }, + { url = "https://files.pythonhosted.org/packages/7d/5c/b863a2c093c4d6f21a597fcaf24ead0835c09ab16a8312d5a5a8868af683/numpy-2.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:5ec3753760c1a6d8bb91200666e545c3a9728e6269dfb5d6ce02340996698aa3", size = 17059053, upload-time = "2026-10-10T20:04:44.976Z" }, + { url = "https://files.pythonhosted.org/packages/0a/60/ced4f57f9a1258a0af74f17cb0b0c2700b5c67cd6678823c803b263e4df3/numpy-2.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b1185012870173de7ae33d370bd45b1cf5baee747ea4b97036b65f4e93016877", size = 18477406, upload-time = "2026-10-10T20:04:47.863Z" }, + { url = "https://files.pythonhosted.org/packages/f9/bd/0ef22dafaafcc7d4bb3ca26b8d2afbd55dedad8eaba99a8c864e1997456f/numpy-2.5.4-cp315-cp315-win32.whl", hash = "sha256:298eca75243f2cbbfdb460560b9fb2a1792a33cf2ab4286efd43d92e8d3df508", size = 6185133, upload-time = "2026-10-10T20:04:50.467Z" }, + { url = "https://files.pythonhosted.org/packages/50/bc/d2651b155ecc608a77e6f4d15495c11f14f19bb98f8bf0c5b0d38f86dda1/numpy-2.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:332f3378fe077dd850e677ec01bdcc4f22368fb5d50ef10b2c79230b1bf5a592", size = 12703085, upload-time = "2026-10-10T20:04:52.63Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d2/45e404f8abb26fb9eda12b94012936873e827b1be76f2ee7890be128312e/numpy-2.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:d4cccbbc78717966f764cd3af4fb70276fa01fc7a2688af11c78901fa5c04f05", size = 10801451, upload-time = "2026-10-10T20:04:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c3/2ae14e09cfdb67dc187a342e15308a21c15bf4d2071f8079e6aee5fe56dc/numpy-2.5.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:950ea81d57ef070665581b6e1b5f6a029306423cd1739c5b95fe78aa30db6b9d", size = 17097121, upload-time = "2026-10-10T20:04:58.403Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cf/305ae624ef8a039414317224abe9ec9c2fe7ea3c2e1cf204d43ff6b2ffb9/numpy-2.5.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c05ede731b03fb1b7591faca9389ade3267d2bddf1ad8882bb3f2cc5e101694f", size = 12135439, upload-time = "2026-10-10T20:05:01.65Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a8/f75c63813aef95827bb2c0d13b12803016853056e8792c280058cdbfe783/numpy-2.5.4-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:5fbf7141bbfd63aea22f435c9062a032b9ea0082fe9845dad7f021d3f1234e71", size = 5571451, upload-time = "2026-10-10T20:05:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/6f/0f/f17763f983868b5c49b4101ebd7e00760bd1769478a6bb6a8de6e085bbac/numpy-2.5.4-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:3573cd22564692a5b899ec344e5d5b9cc4576f2985b96f22af3564ed54f2710f", size = 6883356, upload-time = "2026-10-10T20:05:06.249Z" }, + { url = "https://files.pythonhosted.org/packages/67/a7/8af04c5a79e047996cfa38854dcfbececdd0343a7c933a46fdd03ef6f5da/numpy-2.5.4-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c109eac9cd439193678f69d70733c1108487546ca8eafc107b510ae10c1aecd", size = 15750991, upload-time = "2026-10-10T20:05:08.376Z" }, + { url = "https://files.pythonhosted.org/packages/57/7a/648254290d0c504faa8f2d07aa206660c728802c781a6f3fc68ab7cb5d71/numpy-2.5.4-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80d6ef6e8620eb2c2b4c4caad50b5935d6db3cde2d51581b55dcc79e14016d1d", size = 16757675, upload-time = "2026-10-10T20:05:11.393Z" }, + { url = "https://files.pythonhosted.org/packages/b8/fe/4a8c3cdb0c70400cfe4c5bec42d3099a5673802a95064614b33e07b82aa1/numpy-2.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:77045a4b175bbf5316ec08003880804336c78f92281a1b72222b274ea85ec5ac", size = 17113846, upload-time = "2026-10-10T20:05:14.49Z" }, + { url = "https://files.pythonhosted.org/packages/1b/7e/619692bb67778702c0e9eb2d468568a7573f4e269386ea61aed01ee4e557/numpy-2.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:0f02a46e49cfb6c73bdb7aea1c0d3461dbae9aba613542b65f657cd3d17b9fab", size = 18522915, upload-time = "2026-10-10T20:05:17.33Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b5/4da41c328788f575838f97a098fe8ca691ebc6f6fd73ad4a262ee40b184d/numpy-2.5.4-cp315-cp315t-win32.whl", hash = "sha256:ad62a416ddcf863bf44bba76fbf6b53366ab0692e294f51cae4b5fbe0d246788", size = 6335804, upload-time = "2026-10-10T20:05:19.921Z" }, + { url = "https://files.pythonhosted.org/packages/98/94/6482ddfa3d312490cb9358f375bf2ad56427dbea8769187158e94d653753/numpy-2.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:38f47be9f74ab870d2633b5456ae519c43758a8d1fd05342f0ce4ecc034396ee", size = 12890095, upload-time = "2026-10-10T20:05:21.875Z" }, + { url = "https://files.pythonhosted.org/packages/48/7f/c2d1b436b6e7cfebac140c2579a298344b85f2991a2ce5c3615cefb29400/numpy-2.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:7a14a461d9340f1b46b8648578aed9cdb8b3b018a8fac6c1dde2c9192a01a87f", size = 10883718, upload-time = "2026-10-10T20:05:28.547Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -612,8 +704,16 @@ wheels = [ name = "pandas" version = "3.0.3" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" } }, { name = "python-dateutil" }, { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] @@ -660,12 +760,80 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, ] +[[package]] +name = "pandas" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/b1/a10c0d1a7e3ce44eb42635771004aa503e3b9ddde0bd39c524dda65f7eff/pandas-3.1.0.tar.gz", hash = "sha256:ecb9b2d8a817b1e1ba1fdb90c0abe9eab31b4fdf2562b7a56b86ac66ace17e49", size = 5933421, upload-time = "2026-10-22T11:57:03.087Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/12/b94c28870f351e9650c724e6daedf661455b348a9dca6d717c42c30fa216/pandas-3.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f4772320b7735d96dbbef437129f840dec4175f8a18a2391bd1222ed8943825c", size = 11587014, upload-time = "2026-10-22T11:55:07.208Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4b/af8ca15261db52ab07f5d0c2d4c1bc28d822ea4c2a79756bc2606c012e7c/pandas-3.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cc5e4e12d287ea77bbf4c4d4bf47ed152d82a990d3a6794dee7c350f887532b5", size = 11167712, upload-time = "2026-10-22T11:55:09.523Z" }, + { url = "https://files.pythonhosted.org/packages/49/c4/cd5127751e2bef3d10f07e0d856e120916a3df0df6450f759ddb48c78b1c/pandas-3.1.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b4e696ff70ed06e986feeec09356c491ce878d0df69a36c8e591b6a38afa29b", size = 11476964, upload-time = "2026-10-22T11:55:11.889Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a6/dc89cdd95e12093bc813e9b814a09f1a056be3e1304edaef9bf9dbee108a/pandas-3.1.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9433b5dc2f26ef9d9b56daaf4d41d078bf4372c5d88c42ed5b397a6eea80de16", size = 12019688, upload-time = "2026-10-22T11:55:14.246Z" }, + { url = "https://files.pythonhosted.org/packages/36/87/da14aa671da792612f15847451e731d5dc0f06f6747b76b15f197ffcce13/pandas-3.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cba20fd269808e224d6212273008173978033a1dee0f3f217bfdbd58775a72c", size = 12490149, upload-time = "2026-10-22T11:55:16.675Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a0/1ce996c7354ad37983955571058ba0dde585663163ab067275197183868f/pandas-3.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:743554ed96755ad1963397ccadbe9965d92db8777ca2ae8c1fe58bd9549d7f28", size = 13090624, upload-time = "2026-10-22T11:55:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/92/72/d235f82b51a22478938a3583b68e8dc05811937543adf410f34bab76004a/pandas-3.1.0-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:ac2f833557874a79fc0b5fac4c86e402ea6498beba8753dc29ca23d08481689a", size = 8091628, upload-time = "2026-10-22T11:55:21.285Z" }, + { url = "https://files.pythonhosted.org/packages/10/72/06aefff8fe5efc5e1ca8bfb2695b648a435358f6dc224d4e40159314181a/pandas-3.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:b8c75dbb27c7d170b25c13dd35515da4bb92d40fe8bb3bd2c69a95de2218889b", size = 10748836, upload-time = "2026-10-22T11:55:23.47Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/4c86f659f5b14790d14e17253becc93c4e3d1511b73f0ba2184f06b164f8/pandas-3.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:73b6cea22090b7123c78ee662a5108ff36616b0bec4fe47026f86fb04a49bce7", size = 10128898, upload-time = "2026-10-22T11:55:25.995Z" }, + { url = "https://files.pythonhosted.org/packages/01/54/eb85f8b2ce121c21286842cf81abaa670ec3570173c512298f91128927a5/pandas-3.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:849d5019dead65944875ce21fbf71a2ca18e982164cd7367623cbda8567c3122", size = 11609893, upload-time = "2026-10-22T11:55:28.592Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/b327e814327b7e7e61b5c5cceafd5c0c20f0d38e94f99ebf2dd6c1cec85b/pandas-3.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c1e084760f3b275eed5863407d025b933d25f3247515fcc84bf664405e8bb3d2", size = 11182834, upload-time = "2026-10-22T11:55:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6d/2b918a45ad31d85f898567c948f11d0789ffe7fbadd6f487565e99d179b6/pandas-3.1.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1009f5b6819b81319cf177b4f01a2823ed54c81db8bd11cdf2375f42d064c02", size = 11421478, upload-time = "2026-10-22T11:55:33.257Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6f/b63e1d43eef5746014ae3f4786c72e0020e61b6b373a2abb6ebf1be3df49/pandas-3.1.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de6d0fc96e8551b83f400bf212611a0b3a2b389594d20a2540dacc61b21e7bce", size = 11991052, upload-time = "2026-10-22T11:55:35.556Z" }, + { url = "https://files.pythonhosted.org/packages/fd/43/2cf85145d822957a3f8959fd0a850e6bf687dac2823c612b3912b8888dae/pandas-3.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5611344f9f0ed51619ac7acdbf72b86f5043801a66e3f8730c8971257082e8ad", size = 12455560, upload-time = "2026-10-22T11:55:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/0cb19cea2e6afbb0854cbde0ed1345b61600e0de84eb7b73583fde3e3b55/pandas-3.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:39c67ed9e56ad3ac0727dc2ae6a1eef766988c762d5ce2e2a162613d81c8990b", size = 13045764, upload-time = "2026-10-22T11:55:39.969Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5c/ea5472e8444f5871427f3a6818725684fcdce74223ca1b22aa7ca345d490/pandas-3.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:07807035ad66de0d833d68f9bee05801dbf08b88d56a233b0e9fd189372c3261", size = 10729719, upload-time = "2026-10-22T11:55:42.008Z" }, + { url = "https://files.pythonhosted.org/packages/45/f7/8cdd350c67bc62865d1e30cd7add43dd631420223487ba42dfcfd0d0e052/pandas-3.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:54fed4a50b8321ff8d413b46b7e31a023cecefd06c7210c02aa980a9836428b2", size = 10116651, upload-time = "2026-10-22T11:55:44.367Z" }, + { url = "https://files.pythonhosted.org/packages/c3/a9/e82d44fe20dc61dd2e15bdd17c78aa0023d860d55620e4fe82f9e644a089/pandas-3.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a8c44d40cbb6d00128bf2d8f8bb66c8d561744b6d44686b5a1a01f3b3fd1245f", size = 11675868, upload-time = "2026-10-22T11:55:46.617Z" }, + { url = "https://files.pythonhosted.org/packages/83/38/6be618d98e2cbbb99a40349717532f0c2b6794ddc62f1199e3f83e45a40f/pandas-3.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:89b66823678cdd53cf97187e7cd6861402bdd147d11a383706093136a4e83f45", size = 11262553, upload-time = "2026-10-22T11:55:49.143Z" }, + { url = "https://files.pythonhosted.org/packages/06/5c/66e27fb0d691e835bd55b336bdf47312a541a66a17ba4d72a988d743bb79/pandas-3.1.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a8dc7780720b9e2182799ee86279d4cc7d1fd30b230ea1b202da5b6f7e14c3a", size = 11548155, upload-time = "2026-10-22T11:55:51.674Z" }, + { url = "https://files.pythonhosted.org/packages/37/1e/42e81484c5a6791ab1b8c52c1ff1ef0483c119f710ed58ee603d241b8792/pandas-3.1.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4455cdcc03e42c17233184e8e4ea65c3ed2925d8b1d9d7dd8e6fced3f6e43c1", size = 12029136, upload-time = "2026-10-22T11:55:53.823Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c9/a3b4fd156457f3eb2585838cd56427a113eeeba679204de435a3224b8cc3/pandas-3.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:224a9267e5fa70c16bf4f8a317435eef98387940ce6c8c611f1f68f2c3415e10", size = 12575350, upload-time = "2026-10-22T11:55:56.006Z" }, + { url = "https://files.pythonhosted.org/packages/4e/25/23a017003d792117b297bbd87e35a1bb5658509f610ecd5aeaebb84762aa/pandas-3.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c261c7a808982557c2917d40ffd27916769f26aacb4e6775c01e4467bcd34c3b", size = 13096142, upload-time = "2026-10-22T11:55:58.434Z" }, + { url = "https://files.pythonhosted.org/packages/1c/d2/f4074b7d1234d112aaa23a796be1574441a9fa873707eb98312a237a6185/pandas-3.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:e77112cfcabb3bef7281a72170670db9a5b31409a1cd3199939abb1fba67a7ad", size = 10885958, upload-time = "2026-10-22T11:56:00.895Z" }, + { url = "https://files.pythonhosted.org/packages/25/9f/bcfc9e0ea07f5c7583e129e62cd0529747317977ac1b85878b80235bb322/pandas-3.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:6c51273b89784c9d54020ed2f60e20524169aef349bddf6cba9f5e83b46077a7", size = 10281257, upload-time = "2026-10-22T11:56:03.478Z" }, + { url = "https://files.pythonhosted.org/packages/05/e4/b3bf0b702c0a886e4abcfc2ebdc681310498592a10d9ca0ce5db6df47b70/pandas-3.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:84377fa82d0b40165379a0058d8a1a267ceb09e65160de6fcc75475d4d595884", size = 12134028, upload-time = "2026-10-22T11:56:05.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/ff/9e08cf048d73f4dfdb10da859790cd1a4213465dec39ea367a606d580c4a/pandas-3.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:37f88576f3449bfd86c1052f6b42274b61d7250c8d9dcc55f9dc4de7fe9883ae", size = 11729164, upload-time = "2026-10-22T11:56:08.335Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ff/a0df60d70fac10006841457c30d35647ae0c93af33e420e5e047b15e058d/pandas-3.1.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:756fd67995f3efb6b25f8eaea95c0e893f84e0068e2c53c5fdbb65275041d4db", size = 11413041, upload-time = "2026-10-22T11:56:11.38Z" }, + { url = "https://files.pythonhosted.org/packages/55/04/e6742d85ba7d166a1a8a99ff9f4aa9e585fb24028522daa187859ce50b5d/pandas-3.1.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4d9252b57bded194d09cd5ebf18d9b4edac238b16f56984ea8281d2ecfe39cd", size = 11869427, upload-time = "2026-10-22T11:56:13.637Z" }, + { url = "https://files.pythonhosted.org/packages/0a/a1/c9f4915408ef10b541f31b7812ca038dd903572c163b7be7bdcece8a5110/pandas-3.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:461b79db058639e446ddf90dd7f4804fae385acf0669f6e31321333e14cb6bc5", size = 12430367, upload-time = "2026-10-22T11:56:15.829Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/636b9cc8d30b3e2027b43036dd5a8e9e7cfdd5aa86f4454235c5ef7f96db/pandas-3.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bfac077874a396b7ae61a307b27aa6f0256cf31c143b7f96ac622765754cc300", size = 12945204, upload-time = "2026-10-22T11:56:18.419Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5e/5c28942fa8c6cf2d8b4e941abee43a44ba0a123f9fef6411f0319aefddd5/pandas-3.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1340f823a1d189fae8365de314c4d3c2ea0173c61bca66cc76a91385c64d9d3d", size = 11297553, upload-time = "2026-10-22T11:56:20.934Z" }, + { url = "https://files.pythonhosted.org/packages/cd/a4/5991dfb2cccd233ae754f2fb37a4b1474d90595d83f6790467dc5a7d8041/pandas-3.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2bb3f8cc42fae0c0de5186778f4a4957656b9b3955aca48383abfdac44a83598", size = 10626068, upload-time = "2026-10-22T11:56:23.187Z" }, + { url = "https://files.pythonhosted.org/packages/77/26/5cd42609d8e6cb3991242116a0e499d09f29f5bcf574f72f0a4aba13a87e/pandas-3.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:04ca88727dfa86837030c493d7512bb1c008aa4f5f14149c5f8d3ff446791340", size = 11666949, upload-time = "2026-10-22T11:56:25.842Z" }, + { url = "https://files.pythonhosted.org/packages/97/5f/ca1ac99da700c215c52fb0f5063a7fbf14878c6e100f687704c21985640e/pandas-3.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:c0bfec4d188fb0c05bba47b9f9def87624b8c545913aa3585a28f19f27ce847e", size = 11255062, upload-time = "2026-10-22T11:56:28.201Z" }, + { url = "https://files.pythonhosted.org/packages/d3/eb/8d7aef062e7d1f48869288e5c5ae5d109923f37ed3bbd0fdcaecc87784e9/pandas-3.1.0-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfb087db83eb79ca348047961d367bdd4bf9820d90a5945f3652c7060da7363", size = 11547558, upload-time = "2026-10-22T11:56:30.492Z" }, + { url = "https://files.pythonhosted.org/packages/06/c4/a041c786bc29935afe44806f72c0111f2643cbac5cab761007a227e12e97/pandas-3.1.0-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b5ce18612ca955e7a95d0bacc20120aedf97e2fbbebbe4de029fc24479e79b8", size = 12026259, upload-time = "2026-10-22T11:56:33.219Z" }, + { url = "https://files.pythonhosted.org/packages/d3/4a/ce6690c7e6081c96857ad7dcbe3b978394d8db682610755919e61863516f/pandas-3.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:a13199463b129363701a540c7b79586581b246b10b6a7ad682c41b268e7991d5", size = 12563821, upload-time = "2026-10-22T11:56:35.402Z" }, + { url = "https://files.pythonhosted.org/packages/75/ec/ae440ecbfa8f5ba0230541b4519c1fed0c4293ff362f18f7f353b12e2dca/pandas-3.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:8e4de3fe8017e3445f1174707cd977e4e1154a704af2c060a1f648862f81acb9", size = 13086426, upload-time = "2026-10-22T11:56:37.666Z" }, + { url = "https://files.pythonhosted.org/packages/e7/1e/6459d4b0ed9c3d72b9ab63387ec11e1e658579e702e0ef117be8b5d7d0d1/pandas-3.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:eee72bba8a72f0bf9522132e5c87cd10fce9d77440b50f65326b093ec9fb8d43", size = 10877850, upload-time = "2026-10-22T11:56:39.994Z" }, + { url = "https://files.pythonhosted.org/packages/f5/22/e5fad1828e5f923e80a8d7ba9187f687159004a10f99f95dd5346b04c54e/pandas-3.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:9ea2983fe096a3d997c8c4b9f26775fa51e7fb251b10634bd7f87e6fe3542d52", size = 10276384, upload-time = "2026-10-22T11:56:42.193Z" }, + { url = "https://files.pythonhosted.org/packages/59/76/89820c2e1950425f39d30695d2296a8c7d400c4c2d279e7618363a7289ac/pandas-3.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:84d73e0814a38711e4f2a24b272f748a0b4335169d2f9752407ae26bde22990f", size = 12107257, upload-time = "2026-10-22T11:56:44.498Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2c/af4397e6e855d2c0dcb3392ab7788087be4ef53f6a3f166daad8b83c783e/pandas-3.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d3920d9b4de89fbef76ba6b863d58a330b5bfb720c9f8f58eb9b7da9b711d034", size = 11704430, upload-time = "2026-10-22T11:56:46.791Z" }, + { url = "https://files.pythonhosted.org/packages/fd/9e/7791cd269099f9fca7dc4718955cfe4bb94aedb6634b59eeffefb1ed6a74/pandas-3.1.0-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1d61e0eae5219cd024fcbc21304de759964b5642ddcdbb28d9a8ba442ab6901", size = 11378442, upload-time = "2026-10-22T11:56:49.113Z" }, + { url = "https://files.pythonhosted.org/packages/54/6c/da4013bd33c49a12eb3127771111caeed3222a985651e3465bc06f839ce7/pandas-3.1.0-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbb121205af7bc5e587b94a07b5f67b5ba687a918665b9e8748f21388a88b044", size = 11850193, upload-time = "2026-10-22T11:56:51.558Z" }, + { url = "https://files.pythonhosted.org/packages/14/89/8c820bd6fb6ad8f0edb2877639bc7e164bfb4b8e4cdf7a2202f54959ea1b/pandas-3.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:684ef3add5a92abb99c1622f2bd04b54b8843075a266e8240106ca94ce1d1237", size = 12394022, upload-time = "2026-10-22T11:56:53.785Z" }, + { url = "https://files.pythonhosted.org/packages/92/f5/9d753d85c4ea85b6d1c3735496ae5b416d7ef6905f1845c70e9521d124d9/pandas-3.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dde9a55ecd3c4f9100ac50ab15e852e4c54fcf372d800e4495e0b2159a0122ab", size = 12916085, upload-time = "2026-10-22T11:56:56.084Z" }, + { url = "https://files.pythonhosted.org/packages/e6/f9/4a6965677477dbbb19f8881befb01b67eb6009809db73f7f1b7a0132faf1/pandas-3.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:3fe234bfb9999552dc54c45851752e56e163218d7663897dca950b4a7c32af39", size = 11267097, upload-time = "2026-10-22T11:56:58.479Z" }, + { url = "https://files.pythonhosted.org/packages/99/35/e60fd8228db4eb848802988dbc6e3a0f1a81c57f06ad95ebe4cd534fa655/pandas-3.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:f1a93c24175129bfc9922ef748ddae1d8137c743ed835e2615be329f606fed4d", size = 10597861, upload-time = "2026-10-22T11:57:00.776Z" }, +] + [[package]] name = "pandas-stubs" version = "3.0.0.260204" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/27/1d/297ff2c7ea50a768a2247621d6451abb2a07c0e9be7ca6d36ebe371658e5/pandas_stubs-3.0.0.260204.tar.gz", hash = "sha256:bf9294b76352effcffa9cb85edf0bed1339a7ec0c30b8e1ac3d66b4228f1fbc3", size = 109383, upload-time = "2026-02-04T15:17:17.247Z" } wheels = [ @@ -683,11 +851,11 @@ wheels = [ [[package]] name = "pip" -version = "26.1" +version = "26.2.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/73/7e/d2b04004e1068ad4fdfa2f227b839b5d03e602e47cdbbf49de71137c9546/pip-26.1.tar.gz", hash = "sha256:81e13ebcca3ffa8cc85e4deff5c27e1ee26dea0aa7fc2f294a073ac208806ff3", size = 1840316, upload-time = "2026-04-26T21:00:05.406Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/e8/0842d757e483e895e670f655084cfe8b8c72580c6769a76b30b2789a0608/pip-26.2.2.tar.gz", hash = "sha256:3a8e5c800492787f33a4114ef54444546c340df428974ebba17e0fd47e11af2f", size = 1863336, upload-time = "2026-10-20T16:36:29.008Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/7a/be4bd8bcbb24ea475856dd68159d78b03b2bb53dae369f69c9606b8888f5/pip-26.1-py3-none-any.whl", hash = "sha256:4e8486d821d814b77319acb7b9e8bf5a4ee7590a643e7cb21029f209be8573c1", size = 1812804, upload-time = "2026-04-26T21:00:03.194Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl", hash = "sha256:fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70", size = 1819850, upload-time = "2026-10-20T16:36:27.046Z" }, ] [[package]]