From f5a19f9b24931ee11513690c11ec6e6bceb0be5e Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 8 Sep 2025 09:52:05 +0200 Subject: [PATCH 1/7] Update http.py --- argopy/stores/implementations/http.py | 32 ++++++++++++++++++--------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index a274ea78..23ec30f5 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -300,8 +300,8 @@ def load_lazily( if "ak" not in kwargs: self.ak = ArgoKerchunker() - if self.protocol == 's3': - storage_options = {'anon': not has_aws_credentials()} + if self.protocol == "s3": + storage_options = {"anon": not has_aws_credentials()} else: storage_options = {} self.ak.storage_options = storage_options @@ -314,11 +314,11 @@ def load_lazily( "backend_kwargs": { "consolidated": False, "storage_options": { - "fo": self.ak.to_reference(url, - overwrite=akoverwrite, - fs=self), # codespell:ignore + "fo": self.ak.to_reference( + url, overwrite=akoverwrite, fs=self + ), # codespell:ignore "remote_protocol": fsspec.core.split_protocol(url)[0], - "remote_options": self.ak.storage_options + "remote_options": self.ak.storage_options, }, }, } @@ -327,7 +327,10 @@ def load_lazily( warnings.warn( "This url does not support byte range requests so we cannot load it lazily, falling back on loading in memory." ) - log.debug("This url does not support byte range requests: %s" % self.full_path(url)) + log.debug( + "This url does not support byte range requests: %s" + % self.full_path(url) + ) return load_in_memory( url, errors=errors, dwn_opts=dwn_opts, xr_opts=xr_opts ) @@ -363,7 +366,7 @@ def load_lazily( else: target = target if isinstance(target, bytes) else target.getbuffer() - ds = Dataset(None, memory=target, diskless=True, mode='r') + ds = Dataset(None, memory=target, diskless=True, mode="r") self.register(url) return ds @@ -897,7 +900,9 @@ def read_csv(self, url, **kwargs): self.register(url) return df - def open_json(self, url: str, errors: Literal['raise', 'silent', 'ignore'] = 'raise', **kwargs) -> Any: + def open_json( + self, url: str, errors: Literal["raise", "silent", "ignore"] = "raise", **kwargs + ) -> Any: """Download and process a json document from an url Steps performed: @@ -952,10 +957,15 @@ def open_json(self, url: str, errors: Literal['raise', 'silent', 'ignore'] = 'ra js = json.loads(data, **js_opts) if len(js) == 0: if errors == "raise": - raise DataNotFound("No data loadable from %s, although the url return some data: '%s'" % (url, data)) + raise DataNotFound( + "No data loadable from %s, although the url return some data: '%s'" + % (url, data) + ) elif errors == "ignore": - log.debug("No data loaded from %s, although the url return some data" % url) + log.debug( + "No data loaded from %s, although the url return some data" % url + ) return None else: From 2247150a16efa1788a19a2d2c4d93817d56812fc Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 8 Sep 2025 10:10:21 +0200 Subject: [PATCH 2/7] tidy up the code for debug --- argopy/data_fetchers/erddap_refdata.py | 23 ++++++++++++++++------- argopy/stores/implementations/ftp.py | 8 +++++--- argopy/stores/implementations/http.py | 16 ++++++++++------ argopy/stores/implementations/local.py | 8 +++++--- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/argopy/data_fetchers/erddap_refdata.py b/argopy/data_fetchers/erddap_refdata.py index 635f0df9..d121e7c9 100644 --- a/argopy/data_fetchers/erddap_refdata.py +++ b/argopy/data_fetchers/erddap_refdata.py @@ -1,6 +1,7 @@ """ Fetcher to retrieve ship-based CTD reference data from Ifremer erddap """ + import xarray as xr import logging from ..options import OPTIONS @@ -17,7 +18,9 @@ except: # noqa: E722 # >= v0.8.0 from erddapy.erddapy import ERDDAP # noqa: F401 - from erddapy.erddapy import _quote_string_constraints as quote_string_constraints # noqa: F401 + from erddapy.erddapy import ( + _quote_string_constraints as quote_string_constraints, + ) # noqa: F401 from erddapy.erddapy import parse_dates # noqa: F401 # Soon ! https://github.com/ioos/erddapy @@ -138,7 +141,13 @@ def to_xarray(self, errors: str = "ignore"): # noqa: C901 ) g.append(sub_grp[-1]) ds = xr.concat( - g, dim="N_POINTS", data_vars="minimal", coords="minimal", compat="override" + g, + dim="N_POINTS", + data_vars="minimal", # Only data variables in which the dimension already appears are included. + coords="minimal", # Only coordinates in which the dimension already appears are included. + # If concatenating over a dimension _not_ present in any of the objects, + # then all data variables will be concatenated along that new dimension. + compat="override", # skip comparing and pick variable from first dataset, ) # Cast data types and add variable attributes (not available in the csv download): @@ -163,16 +172,16 @@ def init(self, box: list, **kw): box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max] """ self.BOX = box.copy() - self.definition = ( - "Ifremer erddap ship-based CTD-REFERENCE data fetcher for a space/time region" - ) + self.definition = "Ifremer erddap ship-based CTD-REFERENCE data fetcher for a space/time region" return self def define_constraints(self): """Define request constraints""" - self.erddap.constraints = {"longitude>=": conv_lon(self.BOX[0], conv='180')} - self.erddap.constraints.update({"longitude<=": conv_lon(self.BOX[1], conv='180')}) + self.erddap.constraints = {"longitude>=": conv_lon(self.BOX[0], conv="180")} + self.erddap.constraints.update( + {"longitude<=": conv_lon(self.BOX[1], conv="180")} + ) self.erddap.constraints.update({"latitude>=": self.BOX[2]}) self.erddap.constraints.update({"latitude<=": self.BOX[3]}) self.erddap.constraints.update({"pres>=": self.BOX[4]}) diff --git a/argopy/stores/implementations/ftp.py b/argopy/stores/implementations/ftp.py index 86b2546a..3c33c650 100644 --- a/argopy/stores/implementations/ftp.py +++ b/argopy/stores/implementations/ftp.py @@ -416,9 +416,11 @@ def open_mfdataset( ds = xr.concat( results, dim=concat_dim, - data_vars="minimal", - coords="minimal", - compat="override", + data_vars="minimal", # Only data variables in which the dimension already appears are included. + coords="minimal", # Only coordinates in which the dimension already appears are included. + # If concatenating over a dimension _not_ present in any of the objects, + # then all data variables will be concatenated along that new dimension. + compat="override", # skip comparing and pick variable from first dataset, ) return ds else: diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 23ec30f5..89c25c91 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -514,9 +514,11 @@ def finalize(obj_list, **kwargs): ds = xr.concat( ds_list, dim=concat_dim, - data_vars="minimal", - coords="minimal", - compat="override", + data_vars="minimal", # Only data variables in which the dimension already appears are included. + coords="minimal", # Only coordinates in which the dimension already appears are included. + # If concatenating over a dimension _not_ present in any of the objects, + # then all data variables will be concatenated along that new dimension. + compat="override", # skip comparing and pick variable from first dataset ) log.info("Dataset size after concat: %i" % len(ds[concat_dim])) return ds, True @@ -862,9 +864,11 @@ def open_mfdataset( ds = xr.concat( results, dim=concat_dim, - data_vars="minimal", - coords="minimal", - compat="override", + data_vars="minimal", # Only data variables in which the dimension already appears are included. + coords="minimal", # Only coordinates in which the dimension already appears are included. + # If concatenating over a dimension _not_ present in any of the objects, + # then all data variables will be concatenated along that new dimension. + compat="override", # skip comparing and pick variable from first dataset ) if not compute_details: return ds diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index 81eb98f4..bfba9be7 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -395,9 +395,11 @@ def open_mfdataset( ds = xr.concat( results, dim=concat_dim, - data_vars="minimal", - coords="minimal", - compat="override", + data_vars="minimal", # Only data variables in which the dimension already appears are included. + coords="minimal", # Only coordinates in which the dimension already appears are included. + # If concatenating over a dimension _not_ present in any of the objects, + # then all data variables will be concatenated along that new dimension. + compat="override", # skip comparing and pick variable from first dataset, ) return ds else: From 7103374387f68d2a0553635c8157d666a3a17c16 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 8 Sep 2025 10:29:48 +0200 Subject: [PATCH 3/7] Set xr.concat option `coords` to "all" --- argopy/data_fetchers/erddap_refdata.py | 9 +++++---- argopy/stores/implementations/ftp.py | 7 ++++--- argopy/stores/implementations/http.py | 14 ++++++++------ argopy/stores/implementations/local.py | 7 ++++--- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/argopy/data_fetchers/erddap_refdata.py b/argopy/data_fetchers/erddap_refdata.py index d121e7c9..4f13fa3f 100644 --- a/argopy/data_fetchers/erddap_refdata.py +++ b/argopy/data_fetchers/erddap_refdata.py @@ -141,12 +141,13 @@ def to_xarray(self, errors: str = "ignore"): # noqa: C901 ) g.append(sub_grp[-1]) ds = xr.concat( - g, + g, dim="N_POINTS", data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="minimal", # Only coordinates in which the dimension already appears are included. - # If concatenating over a dimension _not_ present in any of the objects, - # then all data variables will be concatenated along that new dimension. + coords="all", + # coords="minimal", # Only coordinates in which the dimension already appears are included. + # # If concatenating over a dimension _not_ present in any of the objects, + # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset, ) diff --git a/argopy/stores/implementations/ftp.py b/argopy/stores/implementations/ftp.py index 3c33c650..ac882d35 100644 --- a/argopy/stores/implementations/ftp.py +++ b/argopy/stores/implementations/ftp.py @@ -417,9 +417,10 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="minimal", # Only coordinates in which the dimension already appears are included. - # If concatenating over a dimension _not_ present in any of the objects, - # then all data variables will be concatenated along that new dimension. + coords="all", + # coords="minimal", # Only coordinates in which the dimension already appears are included. + # # If concatenating over a dimension _not_ present in any of the objects, + # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset, ) return ds diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index 89c25c91..b8e30b39 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -515,9 +515,10 @@ def finalize(obj_list, **kwargs): ds_list, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="minimal", # Only coordinates in which the dimension already appears are included. - # If concatenating over a dimension _not_ present in any of the objects, - # then all data variables will be concatenated along that new dimension. + coords="all", + # coords="minimal", # Only coordinates in which the dimension already appears are included. + # # If concatenating over a dimension _not_ present in any of the objects, + # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset ) log.info("Dataset size after concat: %i" % len(ds[concat_dim])) @@ -865,9 +866,10 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="minimal", # Only coordinates in which the dimension already appears are included. - # If concatenating over a dimension _not_ present in any of the objects, - # then all data variables will be concatenated along that new dimension. + coords="all", + # coords="minimal", # Only coordinates in which the dimension already appears are included. + # # If concatenating over a dimension _not_ present in any of the objects, + # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset ) if not compute_details: diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index bfba9be7..11ecf2fc 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -396,9 +396,10 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="minimal", # Only coordinates in which the dimension already appears are included. - # If concatenating over a dimension _not_ present in any of the objects, - # then all data variables will be concatenated along that new dimension. + coords="all", + # coords="minimal", # Only coordinates in which the dimension already appears are included. + # # If concatenating over a dimension _not_ present in any of the objects, + # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset, ) return ds From 916275044e074f2d0769801ebed09cee043c06ac Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 8 Sep 2025 11:02:55 +0200 Subject: [PATCH 4/7] change xr.concat option coords from 'minimal' to 'all' --- argopy/data_fetchers/erddap_refdata.py | 3 ++- argopy/stores/implementations/ftp.py | 6 ++---- argopy/stores/implementations/http.py | 9 ++------- argopy/stores/implementations/local.py | 7 ++----- argopy/tests/test_stores_fsspec.py | 9 +++++++++ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/argopy/data_fetchers/erddap_refdata.py b/argopy/data_fetchers/erddap_refdata.py index 4f13fa3f..d4b88a60 100644 --- a/argopy/data_fetchers/erddap_refdata.py +++ b/argopy/data_fetchers/erddap_refdata.py @@ -144,7 +144,8 @@ def to_xarray(self, errors: str = "ignore"): # noqa: C901 g, dim="N_POINTS", data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="all", + coords="all", # All coordinate variables will be concatenated, except those corresponding + # to other dimensions. # coords="minimal", # Only coordinates in which the dimension already appears are included. # # If concatenating over a dimension _not_ present in any of the objects, # # then all data variables will be concatenated along that new dimension. diff --git a/argopy/stores/implementations/ftp.py b/argopy/stores/implementations/ftp.py index ac882d35..ad67fb05 100644 --- a/argopy/stores/implementations/ftp.py +++ b/argopy/stores/implementations/ftp.py @@ -417,10 +417,8 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="all", - # coords="minimal", # Only coordinates in which the dimension already appears are included. - # # If concatenating over a dimension _not_ present in any of the objects, - # # then all data variables will be concatenated along that new dimension. + coords="all", # All coordinate variables will be concatenated, except those corresponding + # to other dimensions. compat="override", # skip comparing and pick variable from first dataset, ) return ds diff --git a/argopy/stores/implementations/http.py b/argopy/stores/implementations/http.py index b8e30b39..1e2f3b3d 100644 --- a/argopy/stores/implementations/http.py +++ b/argopy/stores/implementations/http.py @@ -516,9 +516,6 @@ def finalize(obj_list, **kwargs): dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. coords="all", - # coords="minimal", # Only coordinates in which the dimension already appears are included. - # # If concatenating over a dimension _not_ present in any of the objects, - # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset ) log.info("Dataset size after concat: %i" % len(ds[concat_dim])) @@ -866,10 +863,8 @@ def open_mfdataset( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="all", - # coords="minimal", # Only coordinates in which the dimension already appears are included. - # # If concatenating over a dimension _not_ present in any of the objects, - # # then all data variables will be concatenated along that new dimension. + coords="all", # All coordinate variables will be concatenated, except those corresponding + # to other dimensions. compat="override", # skip comparing and pick variable from first dataset ) if not compute_details: diff --git a/argopy/stores/implementations/local.py b/argopy/stores/implementations/local.py index 11ecf2fc..9bbd7b07 100644 --- a/argopy/stores/implementations/local.py +++ b/argopy/stores/implementations/local.py @@ -391,15 +391,12 @@ def open_mfdataset( results = [r for r in results if r is not None] # Only keep non-empty results if len(results) > 0: if concat: - # ds = xr.concat(results, dim=concat_dim, data_vars='all', coords='all', compat='override') ds = xr.concat( results, dim=concat_dim, data_vars="minimal", # Only data variables in which the dimension already appears are included. - coords="all", - # coords="minimal", # Only coordinates in which the dimension already appears are included. - # # If concatenating over a dimension _not_ present in any of the objects, - # # then all data variables will be concatenated along that new dimension. + coords="all", # All coordinate variables will be concatenated, except those corresponding + # to other dimensions. compat="override", # skip comparing and pick variable from first dataset, ) return ds diff --git a/argopy/tests/test_stores_fsspec.py b/argopy/tests/test_stores_fsspec.py index 3d58d972..1b0ba8fd 100644 --- a/argopy/tests/test_stores_fsspec.py +++ b/argopy/tests/test_stores_fsspec.py @@ -117,6 +117,9 @@ def test_open_mfdataset(self, params): ds = self.fs.open_mfdataset(uri, method=method, progress='disable' if progress else False, concat=concat) if concat: assert isinstance(ds, xr.Dataset) + assert len(ds.coords) == 0 + assert len(ds.data_vars) == 64 + assert ds.sizes == {'N_PROF': 1, 'N_PARAM': 3, 'N_LEVELS': 55, 'N_CALIB': 1, 'N_HISTORY': 10} else: assert is_list_of_datasets(ds) @@ -323,6 +326,9 @@ def test_open_mfdataset(self, params): ) if concat: assert isinstance(ds, xr.Dataset) + assert len(ds.coords) == 0 + assert len(ds.data_vars) == 64 + assert ds.sizes == {'N_PROF': 1, 'N_PARAM': 3, 'N_LEVELS': 71, 'N_CALIB': 1, 'N_HISTORY': 8} else: assert is_list_of_datasets(ds) @@ -535,6 +541,9 @@ def test(this_params): ) if concat: assert isinstance(ds, xr.Dataset) + assert len(ds.coords) == 0 + assert len(ds.data_vars) == 64 + assert ds.sizes == {'N_PROF': 1, 'N_PARAM': 3, 'N_LEVELS': 71, 'N_CALIB': 1, 'N_HISTORY': 8} else: assert is_list_of_datasets(ds) From 8bb7b6da7227aeb4740f484a0f39d641519f01c7 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 8 Sep 2025 11:15:48 +0200 Subject: [PATCH 5/7] Upstream CI tests to use xarray >=2025.8.0 --- ci/requirements/py3.11-all-free.yml | 2 +- ci/requirements/py3.11-core-free.yml | 2 +- ci/requirements/py3.12-all-free.yml | 2 +- ci/requirements/py3.12-core-free.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/requirements/py3.11-all-free.yml b/ci/requirements/py3.11-all-free.yml index 2ed2d685..b3e9c4cf 100644 --- a/ci/requirements/py3.11-all-free.yml +++ b/ci/requirements/py3.11-all-free.yml @@ -15,7 +15,7 @@ dependencies: - requests - scipy - toolz - - xarray >= 2025.7.0 + - xarray >= 2025.8.0 # EXT.UTIL: - gsw diff --git a/ci/requirements/py3.11-core-free.yml b/ci/requirements/py3.11-core-free.yml index 08621c1a..40cbb2b3 100644 --- a/ci/requirements/py3.11-core-free.yml +++ b/ci/requirements/py3.11-core-free.yml @@ -15,7 +15,7 @@ dependencies: - requests - scipy - toolz - - xarray >= 2025.7.0 + - xarray >= 2025.8.0 # EXT.UTIL: # - gsw diff --git a/ci/requirements/py3.12-all-free.yml b/ci/requirements/py3.12-all-free.yml index 655cc49c..edbf75fe 100644 --- a/ci/requirements/py3.12-all-free.yml +++ b/ci/requirements/py3.12-all-free.yml @@ -15,7 +15,7 @@ dependencies: - requests - scipy - toolz - - xarray >= 2025.7.0 + - xarray >= 2025.8.0 # EXT.UTIL: - gsw diff --git a/ci/requirements/py3.12-core-free.yml b/ci/requirements/py3.12-core-free.yml index 610a5878..10ea7299 100644 --- a/ci/requirements/py3.12-core-free.yml +++ b/ci/requirements/py3.12-core-free.yml @@ -15,7 +15,7 @@ dependencies: - requests - scipy - toolz - - xarray >= 2025.7.0 + - xarray >= 2025.8.0 # EXT.UTIL: # - gsw From be79eb4f509a398dff659c11d4b30264be158bf1 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Mon, 8 Sep 2025 16:56:02 +0200 Subject: [PATCH 6/7] Update erddap_refdata.py --- argopy/data_fetchers/erddap_refdata.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/argopy/data_fetchers/erddap_refdata.py b/argopy/data_fetchers/erddap_refdata.py index d4b88a60..3f25807a 100644 --- a/argopy/data_fetchers/erddap_refdata.py +++ b/argopy/data_fetchers/erddap_refdata.py @@ -146,9 +146,6 @@ def to_xarray(self, errors: str = "ignore"): # noqa: C901 data_vars="minimal", # Only data variables in which the dimension already appears are included. coords="all", # All coordinate variables will be concatenated, except those corresponding # to other dimensions. - # coords="minimal", # Only coordinates in which the dimension already appears are included. - # # If concatenating over a dimension _not_ present in any of the objects, - # # then all data variables will be concatenated along that new dimension. compat="override", # skip comparing and pick variable from first dataset, ) From ea5b85019fbb90ccf0b12b9bbf405e405656ebad Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 9 Sep 2025 10:48:53 +0200 Subject: [PATCH 7/7] update docs [skip-ci] --- docs/whats-new.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/whats-new.rst b/docs/whats-new.rst index d45f5514..7ef09ed6 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -7,6 +7,14 @@ What's New |pypi dwn| |conda dwn| +Coming up next (unreleased) +--------------------------- + +Internals +--------- + +- **Fix upstream compatibility** whereby xarray >= 2025.8 deprecation cycle for changing default keyword arguments in :meth:`xarray.merge` and :meth:`xarray.concat` would make Argopy to fail with internal data processing, :issue:`521`. (:pr:`504`) by |gmaze|. + v1.3.0 (22 Aug. 2025) ---------------------