Subset operator with kerchunk
https://projectpythia.org/kerchunk-cookbook/notebooks/using_references/Datatree.html
https://guide.cloudnativegeo.org/kerchunk/kerchunk-in-practice.html
Import clisops tools
[1]:
import clisops.utils.dataset_utils as clidu
from clisops.ops.subset import subset
Example with testdata from CEDA
[2]:
import clisops.utils.testing as clite
mini_esgf_data = clite.get_kerchunk_datasets()
mini_esgf_data
[2]:
{'CMIP6_KERCHUNK_HTTPS_OPEN_JSON': 'https://gws-access.jasmin.ac.uk/public/cmip6_prep/eodh-eocis/kc-indexes-cmip6-http-v1/CMIP6.CMIP.MOHC.UKESM1-1-LL.1pctCO2.r1i1p1f2.Amon.tasmax.gn.v20220513.json',
'CMIP6_KERCHUNK_HTTPS_OPEN_ZST': 'https://gws-access.jasmin.ac.uk/public/cmip6_prep/eodh-eocis/kc-indexes-cmip6-http-v1/CMIP6.CMIP.MOHC.UKESM1-1-LL.1pctCO2.r1i1p1f2.Amon.tasmax.gn.v20220513.json.zst'}
Open remote dataset with clisops
[3]:
# using .json
ds = clidu.open_xr_dataset(mini_esgf_data["CMIP6_KERCHUNK_HTTPS_OPEN_JSON"])
ds
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:450, in get_project_base_dir(project)
449 try:
--> 450 return CONFIG[f"project:{project}"]["base_dir"]
451 except KeyError:
KeyError: 'project:https://gws-access'
During handling of the above exception, another exception occurred:
InvalidProject Traceback (most recent call last)
Cell In[3], line 3
1 # using .json
2
----> 3 ds = clidu.open_xr_dataset(mini_esgf_data["CMIP6_KERCHUNK_HTTPS_OPEN_JSON"])
4 ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/utils/dataset_utils.py:440, in open_xr_dataset(dset, **kwargs)
435 return _open_as_kerchunk(dset, **zarr_file_kwargs)
437 else:
438 # Force the value of dset to be a list if not a list or tuple
439 # use force=True to allow all file paths to pass through DatasetMapper
--> 440 dset = dset_to_filepaths(dset, force=True)
442 # If an empty sequence, then raise an Exception
443 if not len(dset):
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:314, in dset_to_filepaths(dset, force)
296 def dset_to_filepaths(dset, force=False):
297 """
298 Get filepaths deduced from input dset.
299
(...) 312 A list of file paths deduced from the input dataset.
313 """
--> 314 mapper = DatasetMapper(dset, force=force)
315 return mapper.files
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:53, in DatasetMapper.__init__(self, dset, project, force)
50 self._data_path = None
51 self._files = []
---> 53 self._parse(force)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:104, in DatasetMapper._parse(self, force)
102 # get base_dir in the case where project has been supplied
103 if not self._base_dir and self._project:
--> 104 self._base_dir = get_project_base_dir(self._project)
106 # if a file, group of files or directory to files - find files
107 if dset.startswith("/") or dset.endswith(".nc"):
108 # if instance of FileMapper
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:452, in get_project_base_dir(project)
450 return CONFIG[f"project:{project}"]["base_dir"]
451 except KeyError:
--> 452 raise InvalidProject("The project supplied is not known.")
InvalidProject: The project supplied is not known.
[4]:
# using .zst with compression
ds = clidu.open_xr_dataset(mini_esgf_data["CMIP6_KERCHUNK_HTTPS_OPEN_ZST"])
ds
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:450, in get_project_base_dir(project)
449 try:
--> 450 return CONFIG[f"project:{project}"]["base_dir"]
451 except KeyError:
KeyError: 'project:https://gws-access'
During handling of the above exception, another exception occurred:
InvalidProject Traceback (most recent call last)
Cell In[4], line 3
1 # using .zst with compression
2
----> 3 ds = clidu.open_xr_dataset(mini_esgf_data["CMIP6_KERCHUNK_HTTPS_OPEN_ZST"])
4 ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/utils/dataset_utils.py:440, in open_xr_dataset(dset, **kwargs)
435 return _open_as_kerchunk(dset, **zarr_file_kwargs)
437 else:
438 # Force the value of dset to be a list if not a list or tuple
439 # use force=True to allow all file paths to pass through DatasetMapper
--> 440 dset = dset_to_filepaths(dset, force=True)
442 # If an empty sequence, then raise an Exception
443 if not len(dset):
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:314, in dset_to_filepaths(dset, force)
296 def dset_to_filepaths(dset, force=False):
297 """
298 Get filepaths deduced from input dset.
299
(...) 312 A list of file paths deduced from the input dataset.
313 """
--> 314 mapper = DatasetMapper(dset, force=force)
315 return mapper.files
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:53, in DatasetMapper.__init__(self, dset, project, force)
50 self._data_path = None
51 self._files = []
---> 53 self._parse(force)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:104, in DatasetMapper._parse(self, force)
102 # get base_dir in the case where project has been supplied
103 if not self._base_dir and self._project:
--> 104 self._base_dir = get_project_base_dir(self._project)
106 # if a file, group of files or directory to files - find files
107 if dset.startswith("/") or dset.endswith(".nc"):
108 # if instance of FileMapper
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:452, in get_project_base_dir(project)
450 return CONFIG[f"project:{project}"]["base_dir"]
451 except KeyError:
--> 452 raise InvalidProject("The project supplied is not known.")
InvalidProject: The project supplied is not known.
Subset remote dataset with clisops
[5]:
outputs = subset(
ds=ds,
time="1900-01-01/1900-12-31",
area=(0.0, 10.0, 175.0, 90.0),
output_type="xarray",
)
print(f"There are {len(outputs)} outputs.")
outputs[0]
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[5], line 2
1 outputs = subset(
----> 2 ds=ds,
3 time="1900-01-01/1900-12-31",
4 area=(0.0, 10.0, 175.0, 90.0),
5 output_type="xarray",
NameError: name 'ds' is not defined
Example from project Pythia
Open pythia dataset with xarray.
Test dataset
[6]:
url = "s3://carbonplan-share/nasa-nex-reference/references_prod/ACCESS-CM2_historical/reference.parquet"
Open remote s3 dataset with xarray open_dataset
[7]:
import xarray as xr
from fsspec.implementations.reference import ReferenceFileSystem
fs = ReferenceFileSystem(
url,
remote_protocol="s3",
target_protocol="s3",
remote_options={"anon": True},
target_options={"anon": True},
lazy=True,
)
ds = xr.open_dataset(
fs.get_mapper(),
engine="zarr",
backend_kwargs={
"consolidated": False,
"zarr_format": 2,
},
chunks={"time": 3},
)
ds
[7]:
<xarray.Dataset> Size: 738GB
Dimensions: (time: 23741, lat: 600, lon: 1440)
Coordinates:
* time (time) datetime64[ns] 190kB 1950-01-01T12:00:00 ... 2014-12-31T1...
* lat (lat) float64 5kB -59.88 -59.62 -59.38 -59.12 ... 89.38 89.62 89.88
* lon (lon) float64 12kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9
Data variables:
hurs (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
huss (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
pr (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
rlds (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
rsds (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
sfcWind (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
tas (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
tasmax (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
tasmin (time, lat, lon) float32 82GB dask.array<chunksize=(3, 600, 1440), meta=np.ndarray>
Attributes: (12/22)
Conventions: CF-1.7
activity: NEX-GDDP-CMIP6
cmip6_institution_id: CSIRO-ARCCSS
cmip6_license: CC-BY-SA 4.0
cmip6_source_id: ACCESS-CM2
contact: Dr. Rama Nemani: rama.nemani@nasa.gov, Dr. Bridget...
... ...
scenario: historical
source: BCSD
title: ACCESS-CM2, r1i1p1f1, historical, global downscale...
tracking_id: 16d27564-470f-41ea-8077-f4cc3efa5bfe
variant_label: r1i1p1f1
version: 1.0Open dataset with xarray open_zarr
[8]:
import fsspec
mapper = fsspec.get_mapper(
"reference://",
fo=url,
target_options={"anon": True},
remote_options={"anon": True},
remote_protocol="s3",
target_protocol="s3",
)
ds = xr.open_zarr(mapper, consolidated=False, zarr_format=2)
ds
[8]:
<xarray.Dataset> Size: 738GB
Dimensions: (time: 23741, lat: 600, lon: 1440)
Coordinates:
* time (time) datetime64[ns] 190kB 1950-01-01T12:00:00 ... 2014-12-31T1...
* lat (lat) float64 5kB -59.88 -59.62 -59.38 -59.12 ... 89.38 89.62 89.88
* lon (lon) float64 12kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9
Data variables:
hurs (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
huss (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
pr (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
rlds (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
rsds (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
sfcWind (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
tas (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
tasmax (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
tasmin (time, lat, lon) float32 82GB dask.array<chunksize=(1, 600, 1440), meta=np.ndarray>
Attributes: (12/22)
Conventions: CF-1.7
activity: NEX-GDDP-CMIP6
cmip6_institution_id: CSIRO-ARCCSS
cmip6_license: CC-BY-SA 4.0
cmip6_source_id: ACCESS-CM2
contact: Dr. Rama Nemani: rama.nemani@nasa.gov, Dr. Bridget...
... ...
scenario: historical
source: BCSD
title: ACCESS-CM2, r1i1p1f1, historical, global downscale...
tracking_id: 16d27564-470f-41ea-8077-f4cc3efa5bfe
variant_label: r1i1p1f1
version: 1.0Open Pythia s3 dataset with clisops
[9]:
options = {
"remote_protocol": "s3",
"target_protocol": "s3",
"remote_options": {"anon": True},
"target_options": {"anon": True},
}
ds = clidu.open_xr_dataset(url, **options)
ds
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[9], line 8
4 "remote_options": {"anon": True},
5 "target_options": {"anon": True},
6 }
7
----> 8 ds = clidu.open_xr_dataset(url, **options)
9 ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/utils/dataset_utils.py:440, in open_xr_dataset(dset, **kwargs)
435 return _open_as_kerchunk(dset, **zarr_file_kwargs)
437 else:
438 # Force the value of dset to be a list if not a list or tuple
439 # use force=True to allow all file paths to pass through DatasetMapper
--> 440 dset = dset_to_filepaths(dset, force=True)
442 # If an empty sequence, then raise an Exception
443 if not len(dset):
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:314, in dset_to_filepaths(dset, force)
296 def dset_to_filepaths(dset, force=False):
297 """
298 Get filepaths deduced from input dset.
299
(...) 312 A list of file paths deduced from the input dataset.
313 """
--> 314 mapper = DatasetMapper(dset, force=force)
315 return mapper.files
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:53, in DatasetMapper.__init__(self, dset, project, force)
50 self._data_path = None
51 self._files = []
---> 53 self._parse(force)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/project_utils.py:146, in DatasetMapper._parse(self, force)
144 # use to data_path to find files if not set already
145 if len(self._files) < 1:
--> 146 self._files = sorted(glob.glob(os.path.join(self._data_path, "*.nc")))
147 print(self._data_path)
148 print(self._files)
File <frozen posixpath>:76, in join(a, *p)
TypeError: expected str, bytes or os.PathLike object, not NoneType
Subset Python S3 dataset with clisops
[10]:
outputs = subset(
ds=ds,
time="2000-01-01/2000-01-31",
area=(0.0, 10.0, 175.0, 90.0),
output_type="xarray",
)
print(f"There are {len(outputs)} outputs.")
outputs[0]
/home/docs/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.12/site-packages/clisops/utils/dataset_utils.py:1853: UserWarning: For coordinate variable 'lon' no bounds can be identified.
warnings.warn(f"For coordinate variable '{coordinate}' no bounds can be identified.")
There are 11 outputs.
[10]:
<xarray.Dataset> Size: 18GB
Dimensions: (time: 2232, lat: 320, lon: 700)
Coordinates:
* time (time) datetime64[ns] 18kB 1950-01-01T12:00:00 ... 1956-02-10T12...
* lat (lat) float64 3kB 10.12 10.38 10.62 10.88 ... 89.38 89.62 89.88
* lon (lon) float64 6kB 0.125 0.375 0.625 0.875 ... 174.4 174.6 174.9
Data variables:
hurs (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
huss (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
pr (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
rlds (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
rsds (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
sfcWind (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
tas (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
tasmax (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
tasmin (time, lat, lon) float32 2GB dask.array<chunksize=(1, 320, 700), meta=np.ndarray>
Attributes: (12/22)
Conventions: CF-1.7
activity: NEX-GDDP-CMIP6
cmip6_institution_id: CSIRO-ARCCSS
cmip6_license: CC-BY-SA 4.0
cmip6_source_id: ACCESS-CM2
contact: Dr. Rama Nemani: rama.nemani@nasa.gov, Dr. Bridget...
... ...
scenario: historical
source: BCSD
title: ACCESS-CM2, r1i1p1f1, historical, global downscale...
tracking_id: 16d27564-470f-41ea-8077-f4cc3efa5bfe
variant_label: r1i1p1f1
version: 1.0