Subsetting
The subset operation makes use of clisops.core.subset to process the datasets and to set the output type and the output file names.
[1]:
from clisops.utils.testing import stratus, XCLIM_TEST_DATA_VERSION, XCLIM_TEST_DATA_REPO_URL,XCLIM_TEST_DATA_CACHE_DIR
Stratus = stratus(repo=XCLIM_TEST_DATA_REPO_URL, branch=XCLIM_TEST_DATA_VERSION, cache_dir=XCLIM_TEST_DATA_CACHE_DIR)
# fetch files locally or from GitHub
tas_files = [
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_203012-205511.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_205512-208011.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_208012-209912.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_209912-212411.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_212412-214911.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_214912-217411.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_217412-219911.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_219912-222411.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_222412-224911.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_224912-227411.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_227412-229911.nc",
"cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_229912-229912.nc"
]
for i, name in enumerate(tas_files):
tas_files[i] = Stratus.fetch(name)
o3_file = Stratus.fetch("cmip6/o3_Amon_GFDL-ESM4_historical_r1i1p1f1_gr1_185001-194912.nc")
# remove previously created example file
import os
if os.path.exists("./output_001.nc"):
os.remove("./output_001.nc")
Downloading file 'cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc' from 'https://raw.githubusercontent.com/Ouranosinc/xclim-testdata/v2024.8.23/data/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc' to '/home/docs/.cache/xclim-testdata/v2024.8.23'.
---------------------------------------------------------------------------
HTTPError Traceback (most recent call last)
Cell In[1], line 22
6 tas_files = [
7 "cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc",
8 "cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_203012-205511.nc",
(...)
19 "cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_229912-229912.nc"
20 ]
21 for i, name in enumerate(tas_files):
---> 22 tas_files[i] = Stratus.fetch(name)
24 o3_file = Stratus.fetch("cmip6/o3_Amon_GFDL-ESM4_historical_r1i1p1f1_gr1_185001-194912.nc")
26 # remove previously created example file
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/pooch/core.py:589, in Pooch.fetch(self, fname, processor, downloader, progressbar)
586 if downloader is None:
587 downloader = choose_downloader(url, progressbar=progressbar)
--> 589 stream_download(
590 url,
591 full_path,
592 known_hash,
593 downloader,
594 pooch=self,
595 retry_if_failed=self.retry_if_failed,
596 )
598 if processor is not None:
599 return processor(str(full_path), action, self)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/pooch/core.py:807, in stream_download(url, fname, known_hash, downloader, pooch, retry_if_failed)
803 try:
804 # Stream the file to a temporary so that we can safely check its
805 # hash before overwriting the original.
806 with temporary_file(path=str(fname.parent)) as tmp:
--> 807 downloader(url, tmp, pooch)
808 hash_matches(tmp, known_hash, strict=True, source=str(fname.name))
809 shutil.move(tmp, str(fname))
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/pooch/downloaders.py:221, in HTTPDownloader.__call__(self, url, output_file, pooch, check_only)
219 try:
220 response = requests.get(url, timeout=timeout, **kwargs)
--> 221 response.raise_for_status()
222 content = response.iter_content(chunk_size=self.chunk_size)
223 total = int(response.headers.get("content-length", 0))
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/requests/models.py:1024, in Response.raise_for_status(self)
1019 http_error_msg = (
1020 f"{self.status_code} Server Error: {reason} for url: {self.url}"
1021 )
1023 if http_error_msg:
-> 1024 raise HTTPError(http_error_msg, response=self)
HTTPError: 403 Client Error: Forbidden for url: https://raw.githubusercontent.com/Ouranosinc/xclim-testdata/v2024.8.23/data/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc
[2]:
from clisops.ops.subset import subset
import xarray as xr
The subset process takes several parameters:
Subsetting Parameters
ds: Union[xr.Dataset, str, Path]
time: Optional[Union[str, TimeParameter]]
area: Optional[
Union[
str,
Tuple[
Union[int, float, str],
Union[int, float, str],
Union[int, float, str],
Union[int, float, str],
],
AreaParameter,
]
]
level: Optional[
Union[
str, LevelParameter
]
]
time_components: Optional[Union[str, Dict, TimeComponentsParameter]]
output_dir: Optional[Union[str, Path]]
output_type: {"netcdf", "nc", "zarr", "xarray"}
split_method: {"time:auto"}
file_namer: {"standard"}
The output is a list containing the outputs in the format selected.
[3]:
ds = xr.open_mfdataset(tas_files, use_cftime=True, combine="by_coords")
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:210, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
209 try:
--> 210 file = self._cache[self._key]
211 except KeyError:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56, in LRUCache.__getitem__(self, key)
55 with self._lock:
---> 56 value = self._cache[key]
57 self._cache.move_to_end(key)
KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/home/docs/checkouts/readthedocs.org/user_builds/clisops/checkouts/stable/docs/notebooks/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '4e8cd65e-e85d-4108-a976-91b369cce2c0']
During handling of the above exception, another exception occurred:
FileNotFoundError Traceback (most recent call last)
Cell In[3], line 1
----> 1 ds = xr.open_mfdataset(tas_files, use_cftime=True, combine="by_coords")
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:983, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
980 open_ = open_dataset
981 getattr_ = getattr
--> 983 datasets = [open_(p, **open_kwargs) for p in paths]
984 closers = [getattr_(ds, "_close") for ds in datasets]
985 if preprocess is not None:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:983, in <listcomp>(.0)
980 open_ = open_dataset
981 getattr_ = getattr
--> 983 datasets = [open_(p, **open_kwargs) for p in paths]
984 closers = [getattr_(ds, "_close") for ds in datasets]
985 if preprocess is not None:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:526, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
514 decoders = _resolve_decoders_kwargs(
515 decode_cf,
516 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...)
522 decode_coords=decode_coords,
523 )
525 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 526 backend_ds = backend.open_dataset(
527 filename_or_obj,
528 drop_variables=drop_variables,
529 **decoders,
530 **kwargs,
531 )
532 ds = _dataset_from_backend_dataset(
533 backend_ds,
534 filename_or_obj,
(...)
542 **kwargs,
543 )
544 return ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:577, in NetCDF4BackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, lock, autoclose)
557 def open_dataset(
558 self,
559 filename_or_obj,
(...)
574 autoclose=False,
575 ):
576 filename_or_obj = _normalize_path(filename_or_obj)
--> 577 store = NetCDF4DataStore.open(
578 filename_or_obj,
579 mode=mode,
580 format=format,
581 group=group,
582 clobber=clobber,
583 diskless=diskless,
584 persist=persist,
585 lock=lock,
586 autoclose=autoclose,
587 )
589 store_entrypoint = StoreBackendEntrypoint()
590 with close_on_error(store):
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:382, in NetCDF4DataStore.open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
376 kwargs = dict(
377 clobber=clobber, diskless=diskless, persist=persist, format=format
378 )
379 manager = CachingFileManager(
380 netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
381 )
--> 382 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:329, in NetCDF4DataStore.__init__(self, manager, group, mode, lock, autoclose)
327 self._group = group
328 self._mode = mode
--> 329 self.format = self.ds.data_model
330 self._filename = self.ds.filepath()
331 self.is_remote = is_remote_uri(self._filename)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:391, in NetCDF4DataStore.ds(self)
389 @property
390 def ds(self):
--> 391 return self._acquire()
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:385, in NetCDF4DataStore._acquire(self, needs_lock)
384 def _acquire(self, needs_lock=True):
--> 385 with self._manager.acquire_context(needs_lock) as root:
386 ds = _nc4_require_group(root, self._group, self._mode)
387 return ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/contextlib.py:137, in _GeneratorContextManager.__enter__(self)
135 del self.args, self.kwds, self.func
136 try:
--> 137 return next(self.gen)
138 except StopIteration:
139 raise RuntimeError("generator didn't yield") from None
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:198, in CachingFileManager.acquire_context(self, needs_lock)
195 @contextlib.contextmanager
196 def acquire_context(self, needs_lock=True):
197 """Context manager for acquiring a file."""
--> 198 file, cached = self._acquire_with_cache_info(needs_lock)
199 try:
200 yield file
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:216, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
214 kwargs = kwargs.copy()
215 kwargs["mode"] = self._mode
--> 216 file = self._opener(*self._args, **kwargs)
217 if self._mode == "w":
218 # ensure file doesn't get overridden when opened again
219 self._mode = "a"
File src/netCDF4/_netCDF4.pyx:2521, in netCDF4._netCDF4.Dataset.__init__()
File src/netCDF4/_netCDF4.pyx:2158, in netCDF4._netCDF4._ensure_nc_success()
FileNotFoundError: [Errno 2] No such file or directory: '/home/docs/checkouts/readthedocs.org/user_builds/clisops/checkouts/stable/docs/notebooks/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc'
Output to xarray
There will only be one output for this example.
[4]:
outputs = subset(
ds=ds,
time="2007-01-01T00:00:00/2200-12-30T00:00:00",
area=(0.0, 10.0, 175.0, 90.0),
output_type="xarray",
)
print(f"There is only {len(outputs)} output.")
outputs[0]
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[4], line 2
1 outputs = subset(
----> 2 ds=ds,
3 time="2007-01-01T00:00:00/2200-12-30T00:00:00",
4 area=(0.0, 10.0, 175.0, 90.0),
5 output_type="xarray",
6 )
8 print(f"There is only {len(outputs)} output.")
9 outputs[0]
NameError: name 'ds' is not defined
Output to netCDF with simple namer
There is only one output as the file size is under the memory limit so does not need to be split. This example uses the simple namer which numbers output files.
[5]:
outputs = subset(
ds=ds,
time="2007-01-01T00:00:00/2200-12-30T00:00:00",
area=(0.0, 10.0, 175.0, 90.0),
output_type="nc",
output_dir=".",
split_method="time:auto",
file_namer="simple"
)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[5], line 2
1 outputs = subset(
----> 2 ds=ds,
3 time="2007-01-01T00:00:00/2200-12-30T00:00:00",
4 area=(0.0, 10.0, 175.0, 90.0),
5 output_type="nc",
6 output_dir=".",
7 split_method="time:auto",
8 file_namer="simple"
9 )
NameError: name 'ds' is not defined
[6]:
# To open the file
subset_ds = xr.open_mfdataset("./output_001.nc", use_cftime=True, combine="by_coords")
subset_ds
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
Cell In[6], line 3
1 # To open the file
----> 3 subset_ds = xr.open_mfdataset("./output_001.nc", use_cftime=True, combine="by_coords")
4 subset_ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:948, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
945 paths = [os.fspath(p) if isinstance(p, os.PathLike) else p for p in paths]
947 if not paths:
--> 948 raise OSError("no files to open")
950 if combine == "nested":
951 if isinstance(concat_dim, (str, DataArray)) or concat_dim is None:
OSError: no files to open
Output to netCDF with standard namer
There is only one output as the file size is under the memory limit so does not need to be split. This example uses the standard namer which names output filesa ccording the the input file and how it has been subsetted.
[7]:
outputs = subset(
ds=ds,
time="2007-01-01T00:00:00/2200-12-30T00:00:00",
area=(0.0, 10.0, 175.0, 90.0),
output_type="nc",
output_dir=".",
split_method="time:auto",
file_namer="standard"
)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[7], line 2
1 outputs = subset(
----> 2 ds=ds,
3 time="2007-01-01T00:00:00/2200-12-30T00:00:00",
4 area=(0.0, 10.0, 175.0, 90.0),
5 output_type="nc",
6 output_dir=".",
7 split_method="time:auto",
8 file_namer="standard"
9 )
NameError: name 'ds' is not defined
Subsetting by level
[8]:
ds = xr.open_dataset(o3_file, use_cftime=True)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[8], line 1
----> 1 ds = xr.open_dataset(o3_file, use_cftime=True)
NameError: name 'o3_file' is not defined
No subsetting applied
[9]:
result = subset(ds=ds,
output_type="xarray")
result[0].coords
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[9], line 1
----> 1 result = subset(ds=ds,
2 output_type="xarray")
4 result[0].coords
NameError: name 'ds' is not defined
Subsetting over level
[10]:
# subsetting over pressure level (plev)
result = subset(ds=ds,
level="600/100",
output_type="xarray")
print(result[0].coords)
print(f"\nplev has been subsetted and now only has {len(result[0].coords)} values.")
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[10], line 3
1 # subsetting over pressure level (plev)
----> 3 result = subset(ds=ds,
4 level="600/100",
5 output_type="xarray")
7 print(result[0].coords)
8 print(f"\nplev has been subsetted and now only has {len(result[0].coords)} values.")
NameError: name 'ds' is not defined
Use time components
[11]:
ds = xr.open_mfdataset(tas_files, use_cftime=True, combine="by_coords")
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:210, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
209 try:
--> 210 file = self._cache[self._key]
211 except KeyError:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56, in LRUCache.__getitem__(self, key)
55 with self._lock:
---> 56 value = self._cache[key]
57 self._cache.move_to_end(key)
KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/home/docs/checkouts/readthedocs.org/user_builds/clisops/checkouts/stable/docs/notebooks/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '656e99f0-214e-4155-914d-de9081275ea9']
During handling of the above exception, another exception occurred:
FileNotFoundError Traceback (most recent call last)
Cell In[11], line 1
----> 1 ds = xr.open_mfdataset(tas_files, use_cftime=True, combine="by_coords")
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:983, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
980 open_ = open_dataset
981 getattr_ = getattr
--> 983 datasets = [open_(p, **open_kwargs) for p in paths]
984 closers = [getattr_(ds, "_close") for ds in datasets]
985 if preprocess is not None:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:983, in <listcomp>(.0)
980 open_ = open_dataset
981 getattr_ = getattr
--> 983 datasets = [open_(p, **open_kwargs) for p in paths]
984 closers = [getattr_(ds, "_close") for ds in datasets]
985 if preprocess is not None:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:526, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
514 decoders = _resolve_decoders_kwargs(
515 decode_cf,
516 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...)
522 decode_coords=decode_coords,
523 )
525 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 526 backend_ds = backend.open_dataset(
527 filename_or_obj,
528 drop_variables=drop_variables,
529 **decoders,
530 **kwargs,
531 )
532 ds = _dataset_from_backend_dataset(
533 backend_ds,
534 filename_or_obj,
(...)
542 **kwargs,
543 )
544 return ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:577, in NetCDF4BackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, lock, autoclose)
557 def open_dataset(
558 self,
559 filename_or_obj,
(...)
574 autoclose=False,
575 ):
576 filename_or_obj = _normalize_path(filename_or_obj)
--> 577 store = NetCDF4DataStore.open(
578 filename_or_obj,
579 mode=mode,
580 format=format,
581 group=group,
582 clobber=clobber,
583 diskless=diskless,
584 persist=persist,
585 lock=lock,
586 autoclose=autoclose,
587 )
589 store_entrypoint = StoreBackendEntrypoint()
590 with close_on_error(store):
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:382, in NetCDF4DataStore.open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
376 kwargs = dict(
377 clobber=clobber, diskless=diskless, persist=persist, format=format
378 )
379 manager = CachingFileManager(
380 netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
381 )
--> 382 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:329, in NetCDF4DataStore.__init__(self, manager, group, mode, lock, autoclose)
327 self._group = group
328 self._mode = mode
--> 329 self.format = self.ds.data_model
330 self._filename = self.ds.filepath()
331 self.is_remote = is_remote_uri(self._filename)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:391, in NetCDF4DataStore.ds(self)
389 @property
390 def ds(self):
--> 391 return self._acquire()
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:385, in NetCDF4DataStore._acquire(self, needs_lock)
384 def _acquire(self, needs_lock=True):
--> 385 with self._manager.acquire_context(needs_lock) as root:
386 ds = _nc4_require_group(root, self._group, self._mode)
387 return ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/contextlib.py:137, in _GeneratorContextManager.__enter__(self)
135 del self.args, self.kwds, self.func
136 try:
--> 137 return next(self.gen)
138 except StopIteration:
139 raise RuntimeError("generator didn't yield") from None
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:198, in CachingFileManager.acquire_context(self, needs_lock)
195 @contextlib.contextmanager
196 def acquire_context(self, needs_lock=True):
197 """Context manager for acquiring a file."""
--> 198 file, cached = self._acquire_with_cache_info(needs_lock)
199 try:
200 yield file
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:216, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
214 kwargs = kwargs.copy()
215 kwargs["mode"] = self._mode
--> 216 file = self._opener(*self._args, **kwargs)
217 if self._mode == "w":
218 # ensure file doesn't get overridden when opened again
219 self._mode = "a"
File src/netCDF4/_netCDF4.pyx:2521, in netCDF4._netCDF4.Dataset.__init__()
File src/netCDF4/_netCDF4.pyx:2158, in netCDF4._netCDF4._ensure_nc_success()
FileNotFoundError: [Errno 2] No such file or directory: '/home/docs/checkouts/readthedocs.org/user_builds/clisops/checkouts/stable/docs/notebooks/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc'
[12]:
outputs = subset(
ds=ds,
time_components="year: 2010, 2020, 2030|month: 12, 1, 2",
output_type="xarray",
)
print(f"There is only {len(outputs)} output.")
outputs[0]
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[12], line 2
1 outputs = subset(
----> 2 ds=ds,
3 time_components="year: 2010, 2020, 2030|month: 12, 1, 2",
4 output_type="xarray",
5 )
7 print(f"There is only {len(outputs)} output.")
8 outputs[0]
NameError: name 'ds' is not defined
Using parameter classes
[13]:
from clisops.parameter import (
level_interval,
level_series,
time_components,
time_interval,
time_series,
)
[14]:
ds = xr.open_mfdataset(tas_files, use_cftime=True, combine="by_coords")
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:210, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
209 try:
--> 210 file = self._cache[self._key]
211 except KeyError:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56, in LRUCache.__getitem__(self, key)
55 with self._lock:
---> 56 value = self._cache[key]
57 self._cache.move_to_end(key)
KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/home/docs/checkouts/readthedocs.org/user_builds/clisops/checkouts/stable/docs/notebooks/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '60ca1b04-d054-44d9-a14b-f9371f802564']
During handling of the above exception, another exception occurred:
FileNotFoundError Traceback (most recent call last)
Cell In[14], line 1
----> 1 ds = xr.open_mfdataset(tas_files, use_cftime=True, combine="by_coords")
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:983, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
980 open_ = open_dataset
981 getattr_ = getattr
--> 983 datasets = [open_(p, **open_kwargs) for p in paths]
984 closers = [getattr_(ds, "_close") for ds in datasets]
985 if preprocess is not None:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:983, in <listcomp>(.0)
980 open_ = open_dataset
981 getattr_ = getattr
--> 983 datasets = [open_(p, **open_kwargs) for p in paths]
984 closers = [getattr_(ds, "_close") for ds in datasets]
985 if preprocess is not None:
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/api.py:526, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
514 decoders = _resolve_decoders_kwargs(
515 decode_cf,
516 open_backend_dataset_parameters=backend.open_dataset_parameters,
(...)
522 decode_coords=decode_coords,
523 )
525 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 526 backend_ds = backend.open_dataset(
527 filename_or_obj,
528 drop_variables=drop_variables,
529 **decoders,
530 **kwargs,
531 )
532 ds = _dataset_from_backend_dataset(
533 backend_ds,
534 filename_or_obj,
(...)
542 **kwargs,
543 )
544 return ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:577, in NetCDF4BackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, lock, autoclose)
557 def open_dataset(
558 self,
559 filename_or_obj,
(...)
574 autoclose=False,
575 ):
576 filename_or_obj = _normalize_path(filename_or_obj)
--> 577 store = NetCDF4DataStore.open(
578 filename_or_obj,
579 mode=mode,
580 format=format,
581 group=group,
582 clobber=clobber,
583 diskless=diskless,
584 persist=persist,
585 lock=lock,
586 autoclose=autoclose,
587 )
589 store_entrypoint = StoreBackendEntrypoint()
590 with close_on_error(store):
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:382, in NetCDF4DataStore.open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
376 kwargs = dict(
377 clobber=clobber, diskless=diskless, persist=persist, format=format
378 )
379 manager = CachingFileManager(
380 netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
381 )
--> 382 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:329, in NetCDF4DataStore.__init__(self, manager, group, mode, lock, autoclose)
327 self._group = group
328 self._mode = mode
--> 329 self.format = self.ds.data_model
330 self._filename = self.ds.filepath()
331 self.is_remote = is_remote_uri(self._filename)
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:391, in NetCDF4DataStore.ds(self)
389 @property
390 def ds(self):
--> 391 return self._acquire()
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:385, in NetCDF4DataStore._acquire(self, needs_lock)
384 def _acquire(self, needs_lock=True):
--> 385 with self._manager.acquire_context(needs_lock) as root:
386 ds = _nc4_require_group(root, self._group, self._mode)
387 return ds
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/contextlib.py:137, in _GeneratorContextManager.__enter__(self)
135 del self.args, self.kwds, self.func
136 try:
--> 137 return next(self.gen)
138 except StopIteration:
139 raise RuntimeError("generator didn't yield") from None
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:198, in CachingFileManager.acquire_context(self, needs_lock)
195 @contextlib.contextmanager
196 def acquire_context(self, needs_lock=True):
197 """Context manager for acquiring a file."""
--> 198 file, cached = self._acquire_with_cache_info(needs_lock)
199 try:
200 yield file
File ~/checkouts/readthedocs.org/user_builds/clisops/conda/stable/lib/python3.11/site-packages/xarray/backends/file_manager.py:216, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
214 kwargs = kwargs.copy()
215 kwargs["mode"] = self._mode
--> 216 file = self._opener(*self._args, **kwargs)
217 if self._mode == "w":
218 # ensure file doesn't get overridden when opened again
219 self._mode = "a"
File src/netCDF4/_netCDF4.pyx:2521, in netCDF4._netCDF4.Dataset.__init__()
File src/netCDF4/_netCDF4.pyx:2158, in netCDF4._netCDF4._ensure_nc_success()
FileNotFoundError: [Errno 2] No such file or directory: '/home/docs/checkouts/readthedocs.org/user_builds/clisops/checkouts/stable/docs/notebooks/cmip5/tas_Amon_HadGEM2-ES_rcp85_r1i1p1_200512-203011.nc'
[15]:
outputs = subset(
ds=ds,
time=time_interval("2007-01-01T00:00:00", "2200-12-30T00:00:00"),
time_components=time_components(month=["dec", "jan", "feb"]),
output_type="xarray",
)
print(f"There is only {len(outputs)} output.")
outputs[0]
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[15], line 2
1 outputs = subset(
----> 2 ds=ds,
3 time=time_interval("2007-01-01T00:00:00", "2200-12-30T00:00:00"),
4 time_components=time_components(month=["dec", "jan", "feb"]),
5 output_type="xarray",
6 )
8 print(f"There is only {len(outputs)} output.")
9 outputs[0]
NameError: name 'ds' is not defined