diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c4ecb86..dac24ad 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,3 +15,7 @@ updates: labels: - "new: pull request" - "bot" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 0046f51..1f14364 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -18,6 +18,10 @@ on: - "*" workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: name: "Test session=${{ matrix.session }} Python=${{ matrix.python-version }}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99409a1..9172fdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,28 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +# See https://pre-commit.ci/#configuration +# See https://github.com/scientific-python/cookie#sp-repo-review +ci: + autofix_prs: false + autofix_commit_msg: "style: pre-commit fixes" + autoupdate_commit_msg: "chore: update pre-commit hooks" + autoupdate_schedule: "monthly" + + +# Alphabetised, for lack of a better order. files: | (?x)( - docs\/.+\.py| - lib\/.+\.py| - tests\/.+\.py + \.github/\.*| + pyproject\.toml| + \.pre-commit-config\.yaml| + \./.*\.py| + tests/.+\.py ) + +exclude: | + docs/conf.py # sufficiently different that linting is unhelpful + minimum_pre_commit_version: 1.21.0 repos: @@ -17,50 +33,99 @@ repos: - id: check-added-large-files # Check whether files parse as valid Python. - id: check-ast - # Check for file name conflicts on case-insensitive filesytems. + # Check for file name conflicts on case-insensitive filesystems. - id: check-case-conflict # Check for files that contain merge conflict strings. - id: check-merge-conflict # Check for debugger imports and py37+ `breakpoint()` calls in Python source. - id: debug-statements + # Check TOML file syntax. + - id: check-toml + # Check YAML file syntax. + - id: check-yaml + # Makes sure files end in a newline and only a newline. + # Duplicates Ruff W292 but also works on non-Python files. + - id: end-of-file-fixer + # Replaces or checks mixed line ending. + - id: mixed-line-ending # Don't commit to main branch. - id: no-commit-to-branch + # Trims trailing whitespace. + # Duplicates Ruff W291 but also works on non-Python files. + - id: trailing-whitespace + +# Hooks from all other repos +# In terms of ordering: hooks that modify files should be placed before hooks that do not. + +- repo: https://github.com/adamchainz/blacken-docs + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: 1.20.0 + hooks: + - id: blacken-docs + types: [file, rst] + +- repo: https://github.com/astral-sh/ruff-pre-commit + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: "v0.17.3" + hooks: + - id: ruff-check + types: [file, python] + args: [--fix, --show-fixes] + - id: ruff-format + types: [file, python] + +- repo: https://github.com/aio-libs/sort-all + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: v1.3.0 + hooks: + - id: sort-all + types: [file, python] - repo: https://github.com/codespell-project/codespell + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. rev: "v2.4.3" hooks: - id: codespell types_or: [asciidoc, python, markdown, rst] additional_dependencies: [tomli] -- repo: https://github.com/psf/black-pre-commit-mirror - rev: 26.10.1 +- repo: https://github.com/pre-commit/mirrors-mypy + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: 'v2.4.0' hooks: - - id: black - pass_filenames: false - args: [--config=./pyproject.toml, .] + - id: mypy + exclude: 'noxfile\.py|docs/conf\.py|^tests/' # NB controlling this in pyproject.toml DOES NOT WORK, as MyPy doesn't apply excludes to a passed list of (changed) files -- repo: https://github.com/PyCQA/flake8 - rev: 7.4.1 +- repo: https://github.com/numpy/numpydoc + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: v1.11.0 hooks: - - id: flake8 + - id: numpydoc-validation types: [file, python] -- repo: https://github.com/pycqa/isort - rev: 9.0.2 +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 hooks: - - id: isort - name: isort (python) + - id: rst-backticks + types: [rst] -- repo: https://github.com/asottile/blacken-docs - rev: 1.20.0 +- repo: https://github.com/scientific-python/cookie + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: 2026.08.14 hooks: - - id: blacken-docs - types: [file, rst] + - id: sp-repo-review + additional_dependencies: ["repo-review[cli]"] + args: ["--show=errskip"] -- repo: https://github.com/pycqa/pydocstyle - rev: 6.3.0 +- repo: https://github.com/abravalheri/validate-pyproject + # More exhaustive than Ruff RUF200. + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: "v0.27" hooks: - - id: pydocstyle - args: [--convention=numpy] + - id: validate-pyproject +- repo: https://github.com/zizmorcore/zizmor-pre-commit + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: v1.31.1 + hooks: + - id: zizmor diff --git a/docs/conf.py b/docs/conf.py index f354243..b2e896c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,14 @@ -# noqa -# Configuration file for the Sphinx documentation builder. +# Copyright SciTools contributors # -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +""" +Configuration file for the Sphinx documentation builder. + +This file only contains a selection of the most common options. For a full + list see the documentation: +https://www.sphinx-doc.org/en/master/usage/configuration.html +""" # -- Path setup -------------------------------------------------------------- diff --git a/lib/ncdata/__init__.py b/lib/ncdata/__init__.py index 02d25dd..98e48ea 100644 --- a/lib/ncdata/__init__.py +++ b/lib/ncdata/__init__.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ An abstract representation of Netcdf data with groups, variables + attributes. @@ -14,7 +18,8 @@ from ._core import NameMap, NcAttribute, NcData, NcDimension, NcVariable from ._version import __version__ -__all__ = [ +# N.B. avoiding sorting this, the order is convenient for understanding. +__all__ = [ # noqa: RUF022 "NcData", "NcDimension", "NcVariable", diff --git a/lib/ncdata/_core.py b/lib/ncdata/_core.py index 843e223..f71d59f 100644 --- a/lib/ncdata/_core.py +++ b/lib/ncdata/_core.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Our main classes for representing netCDF data. @@ -12,19 +16,13 @@ """ +from collections.abc import Iterable, Mapping, MutableMapping from functools import wraps from typing import ( Any, - Dict, - Iterable, - Mapping, - MutableMapping, - Optional, - Tuple, Union, ) -import numpy import numpy as np @@ -70,11 +68,10 @@ def add(self, item): If the NameMap has a non-None 'item_type', the added item is type checked. """ - if self.item_type is not None: - if not isinstance(item, self.item_type): - raise TypeError( - f'Item expected to be of type {self.item_type} : got {item}."' - ) + if self.item_type is not None and not isinstance(item, self.item_type): + raise TypeError( + f'Item expected to be of type {self.item_type} : got {item}."' + ) if not hasattr(item, "name"): raise TypeError(f"Item has no '.name' property : {item}.") self[item.name] = item @@ -116,7 +113,7 @@ def rename(self, name: str, new_name: str): >>> mymap.rename('a', 'c') >>> mymap {'c': NcAttribute('c', 'A'), 'qqq': NcAttribute('qqq', 'B'), 'd': NcAttribute('d', 'D')} - """ + """ # noqa: E501 if new_name == name: # skip this case, to avoid removing the item because it matches 'new_name'. pass @@ -132,9 +129,7 @@ def rename(self, name: str, new_name: str): self.addall(items) @classmethod - def from_items( - cls, arg: Union[Iterable, Mapping], item_type=None - ) -> "NameMap": + def from_items(cls, arg: Iterable | Mapping, item_type=None) -> "NameMap": """ Convert an iterable or mapping of items to a NameMap. @@ -186,9 +181,7 @@ def from_items( if ( item_type == NcAttribute and len(arg) > 0 - and not isinstance( - next(iter(arg.values())), NcAttribute - ) + and not isinstance(next(iter(arg.values())), NcAttribute) ): # for attributes only, also allow simple name=value map # for which, convert each value to an NcAttribute @@ -202,9 +195,7 @@ def from_items( elif isinstance(arg, Iterable): result.addall(arg) else: - msg = ( - f"Argument must be an iterable or mapping: got {arg}." - ) + msg = f"Argument must be an iterable or mapping: got {arg}." raise TypeError(msg) return result @@ -231,10 +222,10 @@ class AttrvalsDict(MutableMapping): def __init__(self, attrs: NameMap): self._attrs = attrs - def __getitem__(self, key): # noqa: D105 + def __getitem__(self, key): return self._attrs[key].as_python_value() - def __setitem__(self, key, value): # noqa: D105 + def __setitem__(self, key, value): # Special case : assigning an NcAttribute object is equivalent to assigning # its value (which forces rename). # This is supposed to avoid potential confusion (!) @@ -251,31 +242,31 @@ def __setitem__(self, key, value): # noqa: D105 # # Provide the minimal set of methods required for a MutableMapping # - def __len__(self): # noqa: D105 + def __len__(self): return len(self._attrs) - def __iter__(self): # noqa: D105 + def __iter__(self): return iter(self._attrs) - def __delitem__(self, key): # noqa: D105 + def __delitem__(self, key): self._attrs.__delitem__(key) # # Other useful methods # def __str__(self): - """Provide a string representation. + """ + Provide a string representation. - A convenient representation, somewhat consistent with how we print the attributes - in the printout of an NcData / NcVariable. + A convenient representation, somewhat consistent with how we print the + attributes in the printout of an NcData / NcVariable. """ text = "AttrvalsDict{" if not self._attrs: text += "}" else: lines = [ - f"{attr.name!s}: {attr._print_value()}" - for attr in self._attrs.values() + f"{attr.name!s}: {attr._print_value()}" for attr in self._attrs.values() ] if len(lines) == 1: text += lines[0] + "}" @@ -289,10 +280,10 @@ def __str__(self): # Also really nice-to-have : the "rename" operation, as implemented by NameMap @wraps(NameMap.rename) # N.B. this is *not* missing a docstring, actually.. - def rename(self, name: str, new_name: str): # noqa: D102 + def rename(self, name: str, new_name: str): self._attrs.rename(name, new_name) - def copy(self, deepcopy=True): + def copy(self): """ Make a copy. @@ -300,11 +291,11 @@ def copy(self, deepcopy=True): ------- AttrvalsDict - The copy is 'shallow', in that it contains a NameMap with the *same* NcAttributes - as the original. These are themselves mutable, so the copy will change if the - original content attributes are modified, and vice versa. + The copy is 'shallow', in that it contains a NameMap with the *same* + NcAttributes as the original. These are themselves mutable, so the copy will + change if the original content attributes are modified, and vice versa. - N.B. if a truly independent 'deep' copy is required, you can use `copy.deepcopy`. + N.B. for a truly independent 'deep' copy, use `copy.deepcopy`. """ return self.__class__(self._attrs.copy()) @@ -345,7 +336,7 @@ def get_attrval(self, name: str) -> Any: If no such attribute exists, returns None. .. warning:: - This legacy method is now deprecated, and will be removed in a future release. + This legacy method is deprecated, and will be removed in a future release. """ return self.avals.get(name, None) @@ -355,7 +346,7 @@ def set_attrval(self, name: str, value) -> "NcAttribute": Set the Python value of a named attribute in self.attributes. .. warning:: - This legacy method is now deprecated, and will be removed in a future release. + This legacy method is deprecated, and will be removed in a future release. """ self.avals[name] = value @@ -386,28 +377,22 @@ class NcData(_AttributeAccessMixin): def __init__( self, - name: Optional[str] = None, - dimensions: Union[Mapping, Iterable] = None, - variables: Union[Mapping, Iterable] = None, - attributes: Union[Mapping, Iterable] = None, - groups: Union[Mapping, Iterable] = None, - ): # noqa: D107 + name: str | None = None, + dimensions: Mapping | Iterable | None = None, + variables: Mapping | Iterable | None = None, + attributes: Mapping | Iterable | None = None, + groups: Mapping | Iterable | None = None, + ): #: a group/dataset name (optional) self.name: str = name #: group/dataset dimensions - self.dimensions: Dict[str, "NcDimension"] = as_namemap( - dimensions, NcDimension - ) + self.dimensions: dict[str, NcDimension] = as_namemap(dimensions, NcDimension) #: group/dataset variables - self.variables: Dict[str, "NcVariable"] = as_namemap( - variables, NcVariable - ) + self.variables: dict[str, NcVariable] = as_namemap(variables, NcVariable) #: group/dataset global attributes - self.attributes: Dict[str, "NcAttribute"] = as_namemap( - attributes, NcAttribute - ) + self.attributes: dict[str, NcAttribute] = as_namemap(attributes, NcAttribute) #: sub-groups - self.groups: Dict[str, "NcData"] = as_namemap(groups, NcData) + self.groups: dict[str, NcData] = as_namemap(groups, NcData) def _print_content(self) -> str: """ @@ -417,7 +402,6 @@ def _print_content(self) -> str: class, so it isn't technically an abstract method). This "NcData._print_content()" is called recursively for groups. """ - global _STANDARD_INDENT # noqa: F824 # Define a header line (always a separate line). noname = "<'no-name'>" lines = [f" str: lines += [f"{_STANDARD_INDENT}global attributes:"] # NOTE: #2 show like variable attributes, but *no parent name*. attrs_lines = [ - f":{attr._print_content()}" - for attr in self.attributes.values() + f":{attr._print_content()}" for attr in self.attributes.values() ] lines += _addlines_indent( "\n".join(attrs_lines), _STANDARD_INDENT * 2 @@ -454,7 +437,7 @@ def _print_content(self) -> str: # Join with linefeeds for a simple string result. return "\n".join(lines) - def __str__(self): # noqa: D105 + def __str__(self): return self._print_content() # NOTE: for 'repr', an interpretable literal string is too complex. @@ -500,7 +483,7 @@ def slicer(self, *dim_names): return Slicer(self, *dim_names) - def __getitem__(self, indices): # noqa: D105 + def __getitem__(self, indices): # N.B. Python [,,] syntax ensures there is only ever one argument, # either a single key or a tuple : # : [x] --> keys == x @@ -508,7 +491,7 @@ def __getitem__(self, indices): # noqa: D105 return self.slicer().__getitem__(indices) # Define equality in terms of dataset comparison utility - def __eq__(self, other): # noqa: D105 + def __eq__(self, other): if id(other) == id(self): result = True elif not isinstance(other, NcData): @@ -519,6 +502,10 @@ def __eq__(self, other): # noqa: D105 result = dataset_differences(self, other) == [] return result + # Block hashability : NcData objects are mutable, so they cannot be hashed. + # N.B. this also avoids Ruff PLW1641 - q.v. + __hash__ = None + class NcDimension: """ @@ -527,29 +514,24 @@ class NcDimension: Associates a name with a length, and also an 'unlimited' flag. """ - def __init__( - self, name: str, size: int, unlimited: Optional[bool] = None - ): # noqa: D107 + def __init__(self, name: str, size: int, unlimited: bool | None = None): #: dimension name self.name: str = name #: dimension size (current size, if unlimited) self.size: int = size # N.B. we retain the 'zero size means unlimited' - if size == 0: - unlimited = True - else: - unlimited = bool(unlimited) + unlimited = True if size == 0 else bool(unlimited) #: whether dimension is unlimited self.unlimited: bool = unlimited - def _print_content(self) -> str: # noqa: D105 + def _print_content(self) -> str: str_unlim = " **UNLIMITED**" if self.unlimited else "" return f"{self.name} = {self.size}{str_unlim}" - def __repr__(self): # noqa: D105 + def __repr__(self): str_unlim = ", unlimited=True" if self.unlimited else "" return f"NcDimension({self.name!r}, {self.size}{str_unlim})" - def __str__(self): # noqa: D105 + def __str__(self): return repr(self) def copy(self): @@ -564,6 +546,10 @@ def __eq__(self, other): and self.unlimited == other.unlimited ) + # Block hashability : NcData objects are mutable, so they cannot be hashed. + # N.B. this also avoids Ruff PLW1641 - q.v. + __hash__ = None + class NcVariable(_AttributeAccessMixin): """ @@ -591,13 +577,11 @@ def __init__( # a quoted reference when it's not otherwise needed (and we don't want to # import it). # TODO: remove the flake8 annotation if this gets fixed. - data: Optional[ - Union[np.ndarray, "dask.array.array"] # noqa: F821 - ] = None, + data: Union[np.ndarray, "dask.array.Array"] | None = None, dtype: np.dtype = None, - attributes: Union[Mapping, Iterable] = None, + attributes: Mapping | Iterable | None = None, group: "NcData" = None, - ): + ): # noqa: PLR0913, PLR0917 """ Create a variable. @@ -608,21 +592,21 @@ def __init__( #: variable name self.name: str = name #: variable dimension names (a list of strings, *not* a dict of objects) - self.dimensions: Tuple[str] = tuple(dimensions) + self.dimensions: tuple[str] = tuple(dimensions) if data is not None: if not hasattr(data, "dtype"): data = np.asanyarray(data) dtype = data.dtype - #: variable datatype, as a numpy :class:`numpy.dtype` + #: variable datatype, as a numpy :class:`np.dtype` if dtype is not None and not isinstance(dtype, np.dtype): dtype = np.dtype(dtype) - self.dtype: numpy.dtype = dtype + self.dtype: np.dtype = dtype #: variable data (an array-like, typically a dask or numpy array) self.data = data # Supports lazy, and normally provides a dtype #: variable attributes self.attributes: NameMap = as_namemap(attributes, NcAttribute) #: parent group - self.group: Optional[NcData] = group + self.group: NcData | None = group # # Provide some array-like readonly properties reflected from the data. # @property @@ -634,7 +618,7 @@ def __init__( # return self.data.shape def _print_content(self): - global _STANDARD_INDENT # noqa: F824 + global _STANDARD_INDENT dimstr = ", ".join(self.dimensions) typestr = str(self.dtype) if self.dtype else "" hdr = f""] return "\n".join(lines) - def __str__(self): # noqa: D105 + def __str__(self): return self._print_content() # NOTE: as for NcData, an interpretable 'repr' string is too complex. @@ -677,7 +661,7 @@ def copy(self): return var # Define equality in terms of variable comparison utility - def __eq__(self, other): # noqa: D105 + def __eq__(self, other): if id(other) == id(self): result = True elif not isinstance(other, NcVariable): @@ -688,6 +672,10 @@ def __eq__(self, other): # noqa: D105 result = variable_differences(self, other) == [] return result + # Block hashability : NcData objects are mutable, so they cannot be hashed. + # N.B. this also avoids Ruff PLW1641 - q.v. + __hash__ = None + class NcAttribute: """ @@ -699,7 +687,7 @@ class NcAttribute: In an actual netcdf dataset, a "scalar" is actually just an array of length 1. """ - def __init__(self, name: str, value): # noqa: D107 + def __init__(self, name: str, value): #: attribute name self.name: str = name # Attribute values are arraylike, have dtype @@ -707,7 +695,7 @@ def __init__(self, name: str, value): # noqa: D107 self.value: np.ndarray = value @property - def value(self): # noqa: D102 + def value(self): return self._value @value.setter @@ -715,9 +703,7 @@ def value(self, value): if not hasattr(value, "dtype"): value = np.asanyarray(value) if value.ndim > 1: - raise ValueError( - "Attribute value should only be 0- or 1-dimensional." - ) + raise ValueError("Attribute value should only be 0- or 1-dimensional.") self._value = value def as_python_value(self): @@ -749,10 +735,8 @@ def as_python_value(self): """ result = self.value # Attributes are either scalar or 1-D - if result.ndim >= 2: - raise ValueError( - "Attribute value should only be 0- or 1-dimensional." - ) + if result.ndim >= 2: # noqa: PLR2004 + raise ValueError("Attribute value should only be 0- or 1-dimensional.") if result.shape == (1,): # Reduce the dimension, but *always* return an array scalar. @@ -785,10 +769,10 @@ def _print_value(self): def _print_content(self): return f"{self.name} = {self._print_value()}" - def __repr__(self): # noqa: D105 + def __repr__(self): return f"NcAttribute({self.name!r}, {self._print_value()})" - def __str__(self): # noqa: D105 + def __str__(self): return repr(self) def copy(self): @@ -805,8 +789,10 @@ def __eq__(self, other): v1 = self.value v2 = other.value result = ( - v1.shape == v2.shape - and v1.dtype == v2.dtype - and np.all(v1 == v2) + v1.shape == v2.shape and v1.dtype == v2.dtype and np.all(v1 == v2) ) return result + + # Block hashability : NcData objects are mutable, so they cannot be hashed. + # N.B. this also avoids Ruff PLW1641 - q.v. + __hash__ = None diff --git a/lib/ncdata/dataset_like.py b/lib/ncdata/dataset_like.py index 4d58410..1dad242 100644 --- a/lib/ncdata/dataset_like.py +++ b/lib/ncdata/dataset_like.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. r""" An adaptor layer for :mod:`ncdata` to emulate :mod:`netCDF4`. @@ -30,7 +34,7 @@ """ -from typing import Any, Dict, List +from typing import Any import dask.array as da import netCDF4 @@ -46,7 +50,7 @@ class _Nc4DatalikeWithNcattrs: # N.B. "self._ncdata" is the underlying NcData object : either an NcData or # NcVariable object. - def ncattrs(self) -> List[str]: + def ncattrs(self) -> list[str]: return list(self._ncdata.attributes.keys()) def getncattr(self, attr: str): @@ -83,7 +87,10 @@ def __setattr__(self, attr, value): def set_auto_chartostring(self, value): if bool(value): - msg = "Cannot enable 'auto_chartostring' for Nc4Dataselike or Nc4VariableLike." + msg = ( + "Cannot enable 'auto_chartostring' for Nc4Dataselike " + "or Nc4VariableLike." + ) raise ValueError(msg) @@ -118,15 +125,14 @@ def __init__(self, ncdata: NcData = None): for name, ncvar in self._ncdata.variables.items() } self.dimensions = { - name: Nc4DimensionLike(dim) - for name, dim in self._ncdata.dimensions.items() + name: Nc4DimensionLike(dim) for name, dim in self._ncdata.dimensions.items() } @property def groups(self): # noqa: D102 return None # not supported - def createDimension(self, dimname: str, size: int): # noqa: D102 + def createDimension(self, dimname: str, size: int): # noqa: D102, N802 if dimname in self.dimensions: msg = f'creating duplicate dimension "{dimname}".' raise ValueError(msg) @@ -143,10 +149,10 @@ def createVariable( self, varname: str, datatype: np.dtype, - dimensions: List[str] = (), + dimensions: list[str] = (), fill_value=None, - **kwargs: Dict[str, Any], - ): # noqa: D102 + **kwargs: dict[str, Any], # noqa: ARG002 + ): # noqa: D102, N802 if varname in self.variables: msg = f'creating duplicate variable "{varname}".' raise ValueError(msg) @@ -155,9 +161,7 @@ def createVariable( # N.B. to correctly mirror netCDF4, a variable should be created with all-masked # content. For this we need to decode the dims + work out the shape. # NOTE: simplistic version here, as we don't support groups. - shape = tuple( - self._ncdata.dimensions[dim_name].size for dim_name in dimensions - ) + shape = tuple(self._ncdata.dimensions[dim_name].size for dim_name in dimensions) # Note: initial content is an all-masked lazy array, to avoid allocating space # in memory. This is not assignable, so we cannot mimic partial writes, but # that's okay as we don't currently provide any Nc4VariableLike.__setitem__ : @@ -198,7 +202,8 @@ def filepath() -> str: # noqa: D102 return "" def __lt__(self, other): - """Support comparison operations. + """ + Support comparison operations. Not a real operation: we provide "comparison", just so that Iris can load a list of them. @@ -210,7 +215,8 @@ class Nc4VariableLike(_Nc4DatalikeWithNcattrs): """ An object which contains a :class:`ncdata.NcVariable` and emulates a :class:`netCDF4.Variable`. - The core, contained content object, ``self._ncdata``, is a :class:`~ncdata.NcVariable`. + The core, contained content object, ``self._ncdata``, is a + :class:`~ncdata.NcVariable`. This completely defines the parent object state. The property ``._data_array`` is detected by Iris to do direct data transfer @@ -229,11 +235,11 @@ class Nc4VariableLike(_Nc4DatalikeWithNcattrs): Thus, at present, we do *not* provide any of the ``set_auto_mask/scale/maskandscale()`` methods. - """ + """ # noqa: E501 _local_instance_props = ("_ncdata", "name", "datatype", "_data_array") - def __init__(self, ncvar: NcVariable, datatype: np.dtype): # noqa: D107 + def __init__(self, ncvar: NcVariable, datatype: np.dtype): self._ncdata = ncvar self.name = ncvar.name # Note: datatype must be known at creation, which may be before an actual data @@ -245,9 +251,7 @@ def __init__(self, ncvar: NcVariable, datatype: np.dtype): # noqa: D107 if array is None: # temporary empty data (to correctly support never-written content) # NOTE: significantly, does *not* allocate an actual full array in memory - array = da.ma.masked_array( - da.zeros(self.shape, self.datatype), mask=True - ) + array = da.ma.masked_array(da.zeros(self.shape, self.datatype), mask=True) # Convert from the "inner" raw form to masked-and-scaled, and then back again # (by assigning self._data_array). @@ -296,11 +300,10 @@ def _get_fillvalue(self): * for byte data, there is no netCDF default fill, so the result can be None. """ fv = self._ncdata.avals.get("_FillValue", None) - if fv is None: - if self.dtype.itemsize != 1: - # NOTE: single-byte types have NO default fill-value - dtype_code = self.dtype.str[1:] - fv = netCDF4.default_fillvals[dtype_code] + if fv is None and self.dtype.itemsize != 1: + # NOTE: single-byte types have NO default fill-value + dtype_code = self.dtype.str[1:] + fv = netCDF4.default_fillvals[dtype_code] return fv def _maskandscale_inner_to_apparent(self, array): @@ -358,9 +361,7 @@ def _maskandscale_apparent_to_inner(self, array): # 'meta' argument, but in this case we don't know what 'array' might be, # so we have no (other) generic way to construct a suitable 'meta'. array = da.from_array(array, chunks="auto") - array = da.map_blocks( - self._fill_masked, array, fv, meta=array._meta - ) + array = da.map_blocks(self._fill_masked, array, fv, meta=array._meta) return array @@ -395,7 +396,7 @@ def group(self): # noqa: D102 return self._ncdata.group @property - def dimensions(self) -> List[str]: # noqa: D102 + def dimensions(self) -> list[str]: # noqa: D102 return self._ncdata.dimensions # @@ -451,7 +452,7 @@ def shape(self): # noqa: D102 def size(self): # noqa: D102 return np.prod(self.shape) - def chunking(self): # noqa: D102 + def chunking(self): """ Return chunk sizes. @@ -459,19 +460,20 @@ def chunking(self): # noqa: D102 For now, simply returns ``None``. Could be replaced when required. """ - return None + return class Nc4DimensionLike: """ An object which emulates a :class:`netCDF4.Dimension` object. - The core, contained content object, ``self._ncdata``, is a :class:`ncdata.NcDimension`. + The core, contained content object, ``self._ncdata``, is a + :class:`ncdata.NcDimension`. This completely defines the parent object state. """ - def __init__(self, ncdim: NcDimension): # noqa: D107 + def __init__(self, ncdim: NcDimension): self._ncdata = ncdim @property diff --git a/lib/ncdata/iris.py b/lib/ncdata/iris.py index f20bbd3..5a6d46a 100644 --- a/lib/ncdata/iris.py +++ b/lib/ncdata/iris.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. r""" Interface routines for converting data between ncdata and Iris. @@ -5,7 +9,8 @@ """ -from typing import Any, AnyStr, Dict, Iterable, List, Union +from collections.abc import Iterable +from typing import Any, AnyStr # # NOTE: This uses the :mod:`ncdata.dataset_like` interface ability to mimic a @@ -26,7 +31,7 @@ def to_iris( - ncdata: NcData | List[NcData], **iris_load_kwargs: Dict[AnyStr, Any] + ncdata: NcData | list[NcData], **iris_load_kwargs: dict[AnyStr, Any] ) -> CubeList: """ Read Iris cubes from an :class:`~ncdata.NcData`. @@ -55,7 +60,7 @@ def to_iris( def from_iris( - cubes: Union[Cube, Iterable[Cube]], **iris_save_kwargs: Dict[AnyStr, Any] + cubes: Cube | Iterable[Cube], **iris_save_kwargs: dict[AnyStr, Any] ) -> NcData: """ Create an :class:`~ncdata.NcData` from Iris cubes. diff --git a/lib/ncdata/iris_xarray.py b/lib/ncdata/iris_xarray.py index 4198200..d5fa320 100644 --- a/lib/ncdata/iris_xarray.py +++ b/lib/ncdata/iris_xarray.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. r""" Interface routines for converting data between Xarray and Iris. diff --git a/lib/ncdata/netcdf4.py b/lib/ncdata/netcdf4.py index b70a928..60a6317 100644 --- a/lib/ncdata/netcdf4.py +++ b/lib/ncdata/netcdf4.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Interface routines for converting data between ncdata and netCDF4. @@ -7,7 +11,6 @@ from pathlib import Path from threading import Lock -from typing import Dict, Optional, Union import dask.array as da import netCDF4 as nc @@ -25,8 +28,8 @@ def _to_nc4_group( ncdata: NcData, - nc4object: Union[nc.Dataset, nc.Group], - var_kwargs: Optional[Dict[str, Dict]] = None, + nc4object: nc.Dataset | nc.Group, + var_kwargs: dict[str, dict] | None = None, ) -> None: """ Inner routine supporting ``to_nc4``, and recursive calls for sub-groups. @@ -44,10 +47,7 @@ def _to_nc4_group( for varname, var in ncdata.variables.items(): fillattr = "_FillValue" - if fillattr in var.avals: - fill_value = var.avals[fillattr] - else: - fill_value = None + fill_value = var.avals.get(fillattr, None) kwargs = var_kwargs.get(varname, {}) forbidden_keys = set(_Forbidden_Variable_Kwargs) & set(kwargs) @@ -105,11 +105,10 @@ class _NetCDFDataProxy: Copied from Iris, with some simplifications. """ - __slots__ = ("shape", "dtype", "path", "variable_name", "group_names_path") + # N.B. avoiding sorting this, the order is convenient for understanding. + __slots__ = ("shape", "dtype", "path", "variable_name", "group_names_path") # noqa: RUF023 - def __init__( - self, shape, dtype, filepath, variable_name, group_names_path=None - ): + def __init__(self, shape, dtype, filepath, variable_name, group_names_path=None): self.shape = shape self.dtype = dtype self.path = filepath @@ -160,8 +159,8 @@ def __setstate__(self, state): def to_nc4( ncdata: NcData, - nc4_dataset_or_file: Union[nc.Dataset, Path, str], - var_kwargs: Dict[str, Dict] = None, + nc4_dataset_or_file: nc.Dataset | Path | str, + var_kwargs: dict[str, dict] | None = None, ) -> None: """ Save NcData to a netCDF file. @@ -209,17 +208,18 @@ def to_nc4( nc4ds.close() -def _from_nc4_group(nc4ds: Union[nc.Dataset, nc.Group], dim_chunks) -> NcData: +def _from_nc4_group(nc4ds: nc.Dataset | nc.Group, dim_chunks) -> NcData: """ Inner routine for :func:`from_nc4`. See docstring there. - Provided mainly for recursion into groups, also to keep the dataset open/close separate. + Provided mainly for recursion into groups, also to keep the dataset + open/close separate. """ parent_ds = nc4ds group_names_path = [] while parent_ds.parent is not None: - group_names_path = [parent_ds.name] + group_names_path + group_names_path = [parent_ds.name, *group_names_path] parent_ds = parent_ds.parent ncdata = NcData(name=nc4ds.name) @@ -243,13 +243,13 @@ def _from_nc4_group(nc4ds: Union[nc.Dataset, nc.Group], dim_chunks) -> NcData: # Work out the shape of the variable. # It may refer to dimensions in enclosing groups. - group = parent_ds - dims_map = group.dimensions.copy() + nc4_group = parent_ds + dims_map = nc4_group.dimensions.copy() for group_name in group_names_path: # Inner groups take priority, of course : # their dimensions mask any of the same name in outer groups - group = group.groups[group_name] - dims_map.update(group.dimensions) + nc4_group = nc4_group.groups[group_name] + dims_map.update(nc4_group.dimensions) shape = tuple(dims_map[dimname].size for dimname in var.dimensions) @@ -261,9 +261,7 @@ def _from_nc4_group(nc4ds: Union[nc.Dataset, nc.Group], dim_chunks) -> NcData: group_names_path=group_names_path, ) chunks = [dim_chunks.get(name, "auto") for name in var.dimensions] - var.data = da.from_array( - proxy, chunks=chunks, asarray=True, meta=np.ndarray - ) + var.data = da.from_array(proxy, chunks=chunks, asarray=True, meta=np.ndarray) for attrname in nc4var.ncattrs(): var.avals[attrname] = nc4var.getncattr(attrname) @@ -272,17 +270,15 @@ def _from_nc4_group(nc4ds: Union[nc.Dataset, nc.Group], dim_chunks) -> NcData: ncdata.avals[attrname] = nc4ds.getncattr(attrname) # And finally, groups -- by the magic of recursion ... - for group_name, group in nc4ds.groups.items(): - ncdata.groups[group_name] = _from_nc4_group( - nc4ds.groups[group_name], dim_chunks=dim_chunks - ) + for group_name, nc4_group in nc4ds.groups.items(): + ncdata.groups[group_name] = _from_nc4_group(nc4_group, dim_chunks=dim_chunks) return ncdata def from_nc4( - nc4_dataset_or_file: Union[nc.Dataset, nc.Group, Path, str], - dim_chunks: Dict[str, Union[int, str]] = None, + nc4_dataset_or_file: nc.Dataset | nc.Group | Path | str, + dim_chunks: dict[str, int | str] | None = None, ) -> NcData: """ Load NcData from a :class:`netCDF4.Dataset` or netCDF file. diff --git a/lib/ncdata/threadlock_sharing.py b/lib/ncdata/threadlock_sharing.py index a245fc3..23798de 100644 --- a/lib/ncdata/threadlock_sharing.py +++ b/lib/ncdata/threadlock_sharing.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. r""" Support for "unifying" the thread-safety mechanisms between ncdata and other packages. @@ -86,7 +90,7 @@ def enable_lockshare(iris: bool = False, xarray: bool = False): lock as xarray. """ - global _SHARE_PATCHES + global _SHARE_PATCHES # noqa: PLW0603 # N.B. implement sharing *only if* none already exists if not _SHARE_PATCHES and (iris or xarray): if iris and not xarray: @@ -103,8 +107,7 @@ def enable_lockshare(iris: bool = False, xarray: bool = False): ) patches = [mock.patch(_NCDATA_TARGET, new=XARRAY_LOCK)] - else: - assert iris and xarray + else: # case (iris AND xarray) # set both ncdata AND Iris to use the Xarray lock from xarray.backends.netCDF4_ import ( NETCDF4_PYTHON_LOCK as XARRAY_LOCK, @@ -126,7 +129,7 @@ def disable_lockshare(): Does nothing if no lock share is in operation. """ - global _SHARE_PATCHES + global _SHARE_PATCHES # noqa: PLW0603 for patch in _SHARE_PATCHES: patch.stop() _SHARE_PATCHES = [] diff --git a/lib/ncdata/utils/__init__.py b/lib/ncdata/utils/__init__.py index f44fe95..ec5eb40 100644 --- a/lib/ncdata/utils/__init__.py +++ b/lib/ncdata/utils/__init__.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """General user utility functions.""" from ._compare_nc_datasets import dataset_differences, variable_differences diff --git a/lib/ncdata/utils/_compare_nc_datasets.py b/lib/ncdata/utils/_compare_nc_datasets.py index e123830..aafacec 100644 --- a/lib/ncdata/utils/_compare_nc_datasets.py +++ b/lib/ncdata/utils/_compare_nc_datasets.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Utility for comparing 2 netcdf datasets. @@ -8,7 +12,7 @@ """ from pathlib import Path -from typing import AnyStr, List, Union +from typing import AnyStr from warnings import warn import netCDF4 @@ -19,8 +23,8 @@ def dataset_differences( - dataset_or_path_1: Union[Path, AnyStr, nc.Dataset, NcData], - dataset_or_path_2: Union[Path, AnyStr, nc.Dataset, NcData], + dataset_or_path_1: Path | AnyStr | nc.Dataset | NcData, + dataset_or_path_2: Path | AnyStr | nc.Dataset | NcData, check_names: bool = False, check_dims_order: bool = True, check_dims_unlimited: bool = True, @@ -30,7 +34,7 @@ def dataset_differences( check_var_data: bool = True, show_n_first_different: int = 2, suppress_warnings: bool = False, -) -> List[str]: +) -> list[str]: # noqa: PLR0913, PLR0917 r""" Compare two netcdf datasets. @@ -116,21 +120,15 @@ def dataset_differences( See Also -------- :func:`~ncdata.utils.variable_differences` - """ + """ # noqa: E501 ds1_was_path = not hasattr(dataset_or_path_1, "variables") ds2_was_path = not hasattr(dataset_or_path_2, "variables") ds1, ds2 = None, None try: # convert path-likes to netCDF4.Dataset - if ds1_was_path: - ds1 = nc.Dataset(dataset_or_path_1) - else: - ds1 = dataset_or_path_1 + ds1 = nc.Dataset(dataset_or_path_1) if ds1_was_path else dataset_or_path_1 - if ds2_was_path: - ds2 = nc.Dataset(dataset_or_path_2) - else: - ds2 = dataset_or_path_2 + ds2 = nc.Dataset(dataset_or_path_2) if ds2_was_path else dataset_or_path_2 # NOTE: Both ds1 and ds2 are now *either* NcData *or* netCDF4.Dataset # _isncdata() will be used to distinguish. @@ -158,9 +156,7 @@ def dataset_differences( return errs -def _namelist_differences( - l1, l2, elemname, order_strict=True, suppress_warnings=False -): +def _namelist_differences(l1, l2, elemname, order_strict=True, suppress_warnings=False): errs = [] msg = f"{elemname} do not match: {list(l1)} != {list(l2)}" ok = l1 == l2 @@ -206,7 +202,8 @@ def _attribute_arrays_eq(a1, a2): def _array_element_str(x): - """Make a string representation of a numpy array element (scalar). + """ + Make a string representation of a numpy array element (scalar). Does *not* rely on numpy array printing. Instead converts to an equivalent Python object, and takes str(that). @@ -232,7 +229,8 @@ def _array_element_str(x): def _attribute_str(x): - """Make a string representing an attribute value. + """ + Make a string representing an attribute value. Like the above, not depending on numpy array printing. """ @@ -255,7 +253,7 @@ def _attribute_differences( attrs_order=True, suppress_warnings=False, force_first_attrnames=None, -) -> List[str]: +) -> list[str]: # noqa: PLR0913, PLR0917 """ Compare attribute name lists. @@ -292,11 +290,7 @@ def fix_orders(attrlist): continue attr, attr2 = [ - ( - obj.avals[attrname] - if _isncdata(obj) - else obj.getncattr(attrname) - ) + (obj.avals[attrname] if _isncdata(obj) else obj.getncattr(attrname)) for obj in (obj1, obj2) ] @@ -308,10 +302,7 @@ def fix_orders(attrlist): getattr(attr, "dtype", type(attr)) for attr in (attr, attr2) ] - if all( - isinstance(dt, np.dtype) and dt.kind in "SUb" - for dt in (dtype, dtype2) - ): + if all(isinstance(dt, np.dtype) and dt.kind in "SUb" for dt in (dtype, dtype2)): dtype = dtype2 = "string" if dtype != dtype2: msg = ( @@ -340,8 +331,8 @@ def variable_differences( check_var_data: bool = True, show_n_first_different: int = 2, suppress_warnings: bool = False, - _group_id_string: str = None, -) -> List[str]: + _group_id_string: str | None = None, +) -> list[str]: # noqa: PLR0912, PLR0913, PLR0915, PLR0917 r""" Compare variables. @@ -353,11 +344,13 @@ def variable_differences( variables to compare check_attrs_order : bool, default True If False, no error results from the same contents in a different order, - however unless `suppress_warnings` is True, the error string is issued as a warning. + however unless `suppress_warnings` is True, the error string is issued + as a warning. check_var_data : bool, default True If True, all variable data is also checked for equality. If False, only dtype and shape are compared. - NOTE: comparison of large arrays is done in-memory, so may be highly inefficient. + NOTE: comparison of large arrays is done in-memory, so may be highly + inefficient. show_n_first_different: int, default 2 Number of value differences to display. suppress_warnings : bool, default False @@ -383,10 +376,7 @@ def variable_differences( msg = f"'show_n_diffs' must be >=1 : got {show_n_first_different!r}." raise ValueError(msg) - if v1.name == v2.name: - varname = v1.name - else: - varname = f"{v1.name} / {v2.name}" + varname = v1.name if v1.name == v2.name else f"{v1.name} / {v2.name}" if _group_id_string: var_id_string = f'{_group_id_string} variable "{varname}"' @@ -419,7 +409,7 @@ def variable_differences( def safe_varshape(var): if _isncdata(var): # NcVariable passed - if var.data is None: + if var.data is None: # noqa: SIM108 # Allow for NcVariable.data to be empty shape = None else: @@ -442,20 +432,12 @@ def safe_varshape(var): # data values def _is_strtype(dt): - if dt is None: - result = False - else: - result = dt.kind in "SUb" + result = False if dt is None else dt.kind in "SUb" return result is_str, is_str2 = (_is_strtype(dt) for dt in (dtype, dtype2)) # TODO: is this correct check to allow compare between different dtypes? - if ( - check_var_data - and dims == dims2 - and shape == shape2 - and is_str == is_str2 - ): + if check_var_data and dims == dims2 and shape == shape2 and is_str == is_str2: # NOTE: no attempt to use laziness here. Could be improved. def getdata(var): if _isncdata(var): @@ -465,7 +447,13 @@ def getdata(var): else: # expect var to be an actual netCDF4.Variable # (check for obscure property NOT provided by mimics) - assert hasattr(var, "use_nc_get_vars") + if not hasattr(var, "use_nc_get_vars"): + msg = ( + f"Expected var to be a netCDF4.Variable, or compatible. " + f"Got {var!r}, of type {type(var)}: " + "does not provide '.use_nc_get_vars()'." + ) + raise TypeError(msg) data = var[:] if data is None: @@ -475,18 +463,15 @@ def getdata(var): # Return 0D as 1D, as this makes results simpler to interpret. if data.ndim == 0: data = data.flatten() - assert data.shape == (1,) + # NOTE: data.shape == (1,) + return data data, data2 = (getdata(v) for v in (v1, v2)) - flatdata, flatdata2 = ( - np.asanyarray(arr).flatten() for arr in (data, data2) - ) + flatdata, flatdata2 = (np.asanyarray(arr).flatten() for arr in (data, data2)) # For simpler checking, use flat versions - flat_diff_inds = ( - [] - ) # NB *don't* make this an array, it causes problems + flat_diff_inds = [] # NB *don't* make this an array, it causes problems # Work out whether string : N.B. array type does not ALWAYS match the # variable type, because apparently the scalar content of a *masked* scalar @@ -499,9 +484,7 @@ def getdata(var): # Where data is masked, count mask mismatches and skip those points if any(np.ma.is_masked(arr) for arr in (data, data2)): - mask, mask2 = ( - np.ma.getmaskarray(array) for array in (flatdata, flatdata2) - ) + mask, mask2 = (np.ma.getmaskarray(array) for array in (flatdata, flatdata2)) flat_diff_inds = list(np.where(mask != mask2)[0]) # Replace all masked points to exclude them from unmasked-point checks. either_masked = mask | mask2 @@ -524,14 +507,10 @@ def getdata(var): flat_diff_inds = sorted(flat_diff_inds) n_diffs = len(flat_diff_inds) if n_diffs: - msg = ( - f"{var_id_string} data contents differ, at {n_diffs} points: " - ) + msg = f"{var_id_string} data contents differ, at {n_diffs} points: " ellps = ", ..." if n_diffs > show_n_first_different else "" diffinds = flat_diff_inds[:show_n_first_different] - diffinds = [ - np.unravel_index(ind, shape=data.shape) for ind in diffinds - ] + diffinds = [np.unravel_index(ind, shape=data.shape) for ind in diffinds] diffinds_str = ", ".join( str(tuple([int(ind) for ind in x])) for x in diffinds ) @@ -544,17 +523,14 @@ def getdata(var): ) points_lhs_str = f"[{points_lhs_str}{ellps}]" points_rhs_str = f"[{points_rhs_str}{ellps}]" - msg += ( - f"@INDICES{inds_str}" - f" : LHS={points_lhs_str}, RHS={points_rhs_str}" - ) + msg += f"@INDICES{inds_str} : LHS={points_lhs_str}, RHS={points_rhs_str}" errs.append(msg) return errs def _group_differences( - g1: Union[netCDF4.Dataset, netCDF4.Group], - g2: Union[netCDF4.Dataset, netCDF4.Group], + g1: netCDF4.Dataset | netCDF4.Group, + g2: netCDF4.Dataset | netCDF4.Group, group_id_string: str, dims_order: bool = True, vars_order: bool = True, @@ -565,7 +541,7 @@ def _group_differences( check_names: bool = False, check_unlimited: bool = True, show_n_diffs: int = 2, -) -> List[str]: +) -> list[str]: # noqa: PLR0913, PLR0917 """ Inner routine to compare either whole datasets or subgroups. @@ -573,11 +549,8 @@ def _group_differences( """ errs = [] - if check_names: - if g1.name != g2.name: - errs.append( - f"Datasets have different names: {g1.name!r} != {g2.name!r}." - ) + if check_names and g1.name != g2.name: + errs.append(f"Datasets have different names: {g1.name!r} != {g2.name!r}.") # Compare lists of dimension names dimnames, dimnames2 = [list(grp.dimensions.keys()) for grp in (g1, g2)] errs += _namelist_differences( diff --git a/lib/ncdata/utils/_copy.py b/lib/ncdata/utils/_copy.py index 7ae7c04..0e59acb 100644 --- a/lib/ncdata/utils/_copy.py +++ b/lib/ncdata/utils/_copy.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Utility to copy NcData objects, but not copying any contained data arrays.""" from ncdata import NameMap, NcAttribute, NcData diff --git a/lib/ncdata/utils/_dim_indexing.py b/lib/ncdata/utils/_dim_indexing.py index de08717..060ee5e 100644 --- a/lib/ncdata/utils/_dim_indexing.py +++ b/lib/ncdata/utils/_dim_indexing.py @@ -1,5 +1,10 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +from collections.abc import Mapping from numbers import Number -from typing import Any, Mapping +from typing import Any import dask.array as da import numpy as np @@ -62,9 +67,7 @@ def index_by_dimensions( # Dimension names must occur in the ncdata. dimension = ncdata.dimensions.get(dim_name) if dimension is None: - raise ValueError( - f"Dimension {dim_name!r} is not present in 'ncdata'." - ) + raise ValueError(f"Dimension {dim_name!r} is not present in 'ncdata'.") # Specifically error forbidden key types. if np.array(key).ndim > 1: @@ -72,12 +75,12 @@ def index_by_dimensions( f"Key for dimension {dim_name!r} is multi-dimensional: {key}. " "Multi-dimensional keys are not supported." ) - elif key is Ellipsis: + if key is Ellipsis: raise ValueError( f'Key for dimension {dim_name!r} is Ellipsis / "...": ' "Ellipsis is not supported." ) - elif key in (np.newaxis, None): + if key in (np.newaxis, None): raise ValueError( f"Key for dimension {dim_name!r} is np.newaxis / None: " "New-axis is not supported. " @@ -91,9 +94,7 @@ def index_by_dimensions( if dim_name in var.dimensions: # construct a list of slice objects (i_slicedim,) = [ - i - for i, name in enumerate(var.dimensions) - if name == dim_name + i for i, name in enumerate(var.dimensions) if name == dim_name ] slices = [slice(None) for dim in var.dimensions] slices[i_slicedim] = key @@ -154,7 +155,7 @@ class Slicer: See Also -------- :meth:`index_by_dimensions` - """ + """ # noqa: E501 def __init__(self, ncdata: NcData, *dimension_names: tuple[str]): """ @@ -198,6 +199,6 @@ def __getitem__(self, keys) -> NcData: raise ValueError(msg) # NB too *few* keys is not a problem, since 'zip' truncates for us. - dim_kwargs = {name: key for name, key in zip(self.dim_names, keys)} + dim_kwargs = dict(zip(self.dim_names, keys, strict=False)) return index_by_dimensions(self.ncdata, **dim_kwargs) diff --git a/lib/ncdata/utils/_rename_dim.py b/lib/ncdata/utils/_rename_dim.py index 8f86d6f..e4a2259 100644 --- a/lib/ncdata/utils/_rename_dim.py +++ b/lib/ncdata/utils/_rename_dim.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Utility to rename dimensions.""" from ncdata import NcData @@ -8,13 +12,11 @@ def _rename_dims_in_vars(ncdata: NcData, name_from: str, name_to: str) -> None: for var in ncdata.variables.values(): if name_from in var.dimensions: var.dimensions = tuple( - [ - name_to if name == name_from else name - for name in var.dimensions - ] + [name_to if name == name_from else name for name in var.dimensions] ) - # Also rename in all sub-groups, except where the dimension is redefined ("scope hole"). + # Also rename in all sub-groups, except where the dimension is redefined + # (i.e. inside a "scope hole"). for grp in ncdata.groups.values(): if name_from not in grp.dimensions: _rename_dims_in_vars(grp, name_from, name_to) @@ -34,9 +36,7 @@ def _check_name_collisions(ncdata, name_from, name_to, group_path=""): # Skip this group as its 'name_from' dim makes it a "scope hole". continue inner_path = group_path + "/" + group.name - _check_name_collisions( - group, name_from, name_to, group_path=inner_path - ) + _check_name_collisions(group, name_from, name_to, group_path=inner_path) def rename_dimension(ncdata: NcData, name_from: str, name_to: str) -> None: diff --git a/lib/ncdata/utils/_save_errors.py b/lib/ncdata/utils/_save_errors.py index 5670e8b..f0bbcc2 100644 --- a/lib/ncdata/utils/_save_errors.py +++ b/lib/ncdata/utils/_save_errors.py @@ -1,7 +1,9 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """User utility routines for ncdata.""" -from typing import Dict, List, Union - import netCDF4 as nc import numpy as np @@ -13,12 +15,11 @@ def _name_is_valid(name) -> bool: if not isinstance(name, str) or not name: # Catches non-string (e.g. None, 0, ..) and empty string result = False - else: - # The name rules for netCDF are not fully clear, but seem *extremely* liberal. - # It seems that "/" is not allowed, and that's about it - # So *allow* whitespace, backslash, initial digit, initial underscore ... - if "/" in name: - result = False + # The name rules for netCDF are not fully clear, but seem *extremely* liberal. + # It seems that "/" is not allowed, and that's about it + # So *allow* whitespace, backslash, initial digit, initial underscore ... + elif "/" in name: + result = False return result @@ -33,13 +34,12 @@ def _name_errors(element_container, id_string): ) if not _name_is_valid(name): errors.append( - f"{id_string}s has an element with an invalid netCDF name : " - f"{name!r}" + f"{id_string}s has an element with an invalid netCDF name : {name!r}" ) return errors -_NETCDF_VALID_DTYPES = [np.dtype(key) for key in nc.default_fillvals.keys()] +_NETCDF_VALID_DTYPES = [np.dtype(key) for key in nc.default_fillvals] def _valid_attr_dtype(dtype): @@ -48,9 +48,7 @@ def _valid_attr_dtype(dtype): return dtype.kind in "SU" or dtype in _NETCDF_VALID_DTYPES -def _invalid_attr_errors( - element: Union[NcData, NcVariable], name_prefix: str -) -> List[str]: +def _invalid_attr_errors(element: NcData | NcVariable, name_prefix: str) -> list[str]: errors = [] for attr in element.attributes.values(): dtype = attr.value.dtype @@ -63,8 +61,8 @@ def _invalid_attr_errors( def _variable_errors( - var: NcVariable, var_prefix: str, known_dimensions: Dict[str, int] -) -> List[str]: + var: NcVariable, var_prefix: str, known_dimensions: dict[str, int] +) -> list[str]: errors = [] if var.data is None: errors.append(f"{var_prefix} has no data array.") @@ -98,9 +96,9 @@ def _variable_errors( def _save_errors_inner( ncdata: NcData, - enclosing_dimensions: Dict[str, int] = None, - group_path: str = None, -) -> List[str]: + enclosing_dimensions: dict[str, int] | None = None, + group_path: str | None = None, +) -> list[str]: """ Scan dataset, with context allowing operation over inner groups. @@ -179,7 +177,7 @@ def _save_errors_inner( return errors -def save_errors(ncdata: NcData) -> List[str]: +def save_errors(ncdata: NcData) -> list[str]: """ Scan a dataset for consistency and completeness. @@ -205,7 +203,8 @@ def save_errors(ncdata: NcData) -> List[str]: ----- The checks made are roughly the following: - **(1)** check names in all components (dimensions, variables, attributes and groups): + **(1)** check names in all components + (dimensions, variables, attributes and groups): * all names are valid netcdf names * all element names match their key in the component, diff --git a/lib/ncdata/xarray.py b/lib/ncdata/xarray.py index 2a7306c..b5abe87 100644 --- a/lib/ncdata/xarray.py +++ b/lib/ncdata/xarray.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. r""" Interface routines for converting data between ncdata and xarray. @@ -11,7 +15,7 @@ # from pathlib import Path -from typing import AnyStr, Union +from typing import AnyStr import dask.array as da import numpy as np @@ -67,8 +71,8 @@ def store( self, variables, attributes, - check_encoding_set=frozenset(), - writer=None, + check_encoding_set=frozenset(), # noqa: ARG002 + writer=None, # noqa: ARG002 unlimited_dims=None, ): """ @@ -91,7 +95,7 @@ def store( if varname in self.ncdata.variables: raise ValueError(f'duplicate variable : "{varname}"') - for dim_name, size in zip(var.dims, var.shape): + for dim_name, size in zip(var.dims, var.shape, strict=True): if dim_name in self.ncdata.dimensions: if self.ncdata.dimensions[dim_name].size != size: raise ValueError( @@ -106,8 +110,7 @@ def store( ) attrs = { - name: NcAttribute(name, value) - for name, value in var.attrs.items() + name: NcAttribute(name, value) for name, value in var.attrs.items() } data = var.data @@ -146,14 +149,10 @@ def close(self): # either real or lazy arrays. # @classmethod - def from_xarray( - cls, dataset_or_file: Union[xr.Dataset, AnyStr, Path], **xr_load_kwargs - ): + def from_xarray(cls, dataset_or_file: xr.Dataset | AnyStr | Path, **xr_load_kwargs): if not isinstance(dataset_or_file, xr.Dataset): # It's a "file" (or pathstring, or Path ?). - dataset_or_file = xr.load_dataset( - dataset_or_file, **xr_load_kwargs - ) + dataset_or_file = xr.load_dataset(dataset_or_file, **xr_load_kwargs) nc_data = cls() dataset_or_file.dump_to_store(nc_data, **xr_load_kwargs) return nc_data @@ -189,9 +188,7 @@ def to_xarray(ncdata: NcData, **xarray_load_kwargs) -> xr.Dataset: return _XarrayNcDataStore(ncdata).to_xarray(**xarray_load_kwargs) -def from_xarray( - xrds: Union[xr.Dataset, Path, AnyStr], **xarray_save_kwargs -) -> NcData: +def from_xarray(xrds: xr.Dataset | Path | AnyStr, **xarray_save_kwargs) -> NcData: """ Convert an xarray :class:`xarray.Dataset` to a :class:`NcData`. diff --git a/pyproject.toml b/pyproject.toml index aac3b80..39933e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,29 +1,16 @@ [build-system] # Defined by PEP 518 requires = [ - "setuptools>=77", - "setuptools_scm[toml]>=7.0", - "wheel", + "setuptools>=77.0.3", + "setuptools_scm[toml]>=8", ] # Defined by PEP 517 build-backend = "setuptools.build_meta" [project] -name = "ncdata" authors = [ {name = "Iris Contributors", email = "scitools.pub@gmail.com"} ] -description = "Abstract NetCDF data objects, providing fast data transfer between analysis packages." -requires-python = ">=3.10" -keywords = [ - "cf-metadata", - "data-analysis", - "netcdf", - "iris", - "xarray", -] -license = "BSD-3-Clause" -license-files = ["LICENSE"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", @@ -41,13 +28,26 @@ classifiers = [ "Topic :: Scientific/Engineering", ] dynamic = [ + # NOTE: dependencies are NOT dynamic (for now). "readme", "version", ] +description = "Abstract NetCDF data objects, providing fast data transfer between analysis packages." # Strictly, dask + netcdf are not core dependencies. # But almost *any* conceivable use will need them. # For now, this works in place of a dependencies file, so 'dependencies' is NOT dynamic. dependencies = ["numpy", "dask", "netCDF4"] +keywords = [ + "cf-metadata", + "data-analysis", + "netcdf", + "iris", + "xarray", +] +license = "BSD-3-Clause" +license-files = ["LICENSE"] +name = "ncdata" +requires-python = ">=3.10" [project.urls] @@ -56,37 +56,228 @@ Discussions = "https://github.com/SciTools/ncdata/discussions" Documentation = "https://ncdata.readthedocs.io" Issues = "https://github.com/SciTools/ncdata/issues" +[tool.check-manifest] +# Ignores here will vary by repository. +ignore = ["lib/ncdata/_version.py/_version.py"] + +[tool.codespell] +ignore-words-list = "alpha-numeric,assertIn,degreee,discontiguities,lazyness,meaned,nin,re-use,re-uses,re-using,re-used,anc,abl,ND,Nd" + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.coverage.run] +branch = true +include = [ + "lib/ncdata/*" +] +# NB no omit, sinces tests are not within the main package + + +[tool.mypy] +# https://mypy.readthedocs.io/en/stable/error_code_list2.html +disable_error_code = [] # Include any error codes that you want to ignore +enable_error_code = [ + "ignore-without-code", + "redundant-expr", + "truthy-bool", +] +strict = true +warn_unreachable = true + +[tool.numpydoc_validation] +# We have agreed a that some of the numpydoc rules that we are going to ignore. +# Our repositories are going to have extra rules that they are ignoring that is expected, but they are not included here. +# https://numpydoc.readthedocs.io/en/latest/validation.html#validation-checks +checks = [ + "all", # Enable all numpydoc validation rules, apart from the following: + + # -> Docstring text (summary) should start in the line immediately + # after the opening quotes (not in the same line, or leaving a + # blank line in between) + # Reason for ignore: A chosen house-style + "GL01", # Permit summary line on same line as docstring opening quotes. + + # -> Closing quotes should be placed in the line after the last text + # in the docstring (do not close the quotes in the same line as + # the text, or leave a blank line between the last text and the + # quotes) + # Reason for ignore: A chosen house-style + "GL02", # Permit a blank line before docstring closing quotes. + + # -> Double line break found; please use only one blank line to + # separate sections or paragraphs, and do not leave blank lines + # at the end of docstrings + # Reason for ignore: We benefit from more flexability in formatting + "GL03", # Ignoring. + + # -> See Also section not found + "SA01", # Not all docstrings require a "See Also" section. + + # -> No extended summary found + "ES01", # Not all docstrings require an "Extended Summary" section. + + # -> No examples section found + "EX01", # Not all docstrings require an "Examples" section. + + # -> No Yields section found + "YD01", # Not all docstrings require a "Yields" section. + + # TODO: exceptions that still need investigating are below. Might be fixable, or might become permanent (above): +] +exclude = [ + '\.__eq__$', + '\.__ne__$', + '\.__repr__$', +] +exclude_files = ["^tests/.*", "^lib/ncdata/_version.py$"] + +[tool.pytest.ini_options] # The list of keys vary by repository +addopts = ["-ra", "-v", "--strict-config", "--strict-markers", "--doctest-modules"] # This addopts list varies by repository +filterwarnings = ["default"] +log_level = "INFO" +minversion = "6.0" +testpaths = "" # e.g "cf_units" or "lib/iris" +xfail_strict = "True" + +[tool.repo-review] +# We have agreed that we will permanently ignore the repo-review rules in the first half of this ignore list. +ignore = [ + # https://learn.scientific-python.org/development/guides/style/#PC180 + # Not possible to run on the hardware used by the majority of our developers. Might change in future! + "PC180", # Uses prettier. + # https://learn.scientific-python.org/development/guides/packaging-simple#PP006 + "PP006", # Uses dev dependency group + + # Project-specific. May be reviewed in future. + "PY007", # should have a task runner (nox/tox etc). + + # TODO: exceptions that still need investigating are below. Might be fixable, or might become permanent (above): +] +[tool.ruff] +extend-exclude = [ + "docs/conf.py", # based on Sphix template, is too different to be worth checking. +] +line-length = 88 +src = ["lib"] + +[tool.ruff.format] +exclude = [ + "*.md", "*.rst" #don't reformat code in docs etc +] +preview = false + +[tool.ruff.lint] +# We have agreed that we will permanently ignore the ruff rules in the first half of this ignore list. +ignore = [ + # flake8-commas (COM) + # https://docs.astral.sh/ruff/rules/#flake8-commas-com + "COM812", # Trailing comma missing. + "COM819", # Trailing comma prohibited. + + # flake8-implicit-str-concat (ISC) + # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/ + # NOTE: This rule may cause conflicts when used with "ruff format". + "ISC001", # Implicitly concatenate string literals on one line. + + # project-specific ignores + "ANN", # not needed as we are using MyPy + "C408", # Unnecessary `dict()` call (rewrite as a literal) + # [[TRUST: where used, readabiility is wanted over efficiency]] + "FBT", # flake8-boolean-trap (no bools in positional args) + "FIX002", # TODO comments. + "PLC0415", # Imports at top [[TRUST: exceptions are intentional]] + "SLF", # flake8 private member accesses + "T201", # Print statement. + "TD", # flake8 TODO bans + "RET504", # unnecessary assignment before return [[readabiility wins]] + "TRY003", # Avoid...long messages outside the exception class [[over-applied. unhelpful]] + "EM", # (string construstion in raised exceptions) [[unhelpful]] + "B028", # warn without stacklevel= [[probably not]] + "FLY002", # use f-string instead of joins [[TRUST: readability]] + "RUF015", # prefer 'next' over [0] [[readability]] + "N813", # Camelcase imported as lowercase [[ok for netCDF4]] + "E402", # module level import not at top of file [[TRUST: usages intended]] + + # TODO: exceptions that still need investigating are below. Might be fixable, or might become permanent (above): + "ERA001", # Found commented-out code +] +preview = false +select = [ + "ALL", + # Note: the above "all" disables conflicting rules, if you want that + # rule it needs to be explicitly enabled below: + "D213", # Multi-line docstring summary should start at the second line +] + +[tool.ruff.lint.isort] +force-sort-within-sections = true +#known-first-party = ["ncdata"] # not needed, follows ruff 'src' setting +known-local-folder = ["tests"] + +[tool.ruff.lint.flake8-copyright] +notice-rgx = "^[ #]*Copyright SciTools contributors" + +[tool.ruff.lint.mccabe] +max-complexity = 25 # TODO: reduce, and ignore in specific cases. + +[tool.ruff.lint.per-file-ignores] +# All test scripts +"./tests/**/*.py" = [ # e.g "cf_units/tests" or "lib/iris/tests" + # https://docs.astral.sh/ruff/rules + # Generic ignores. + "D101", # Missing docstring in public class + "D205", # 1 blank line required between summary line and description + "D401", # 1 First line of docstring should be in imperative mood + + # Project-specific + "ARG001", # Unused function argument [[does not recognise explicitly imported pytest fixtures]] + "ARG002", # Unused method argument [[does not recognise explicitly imported pytest fixtures]] + "B018", # useless expression [[used for explicit pytest fixture imports]] + "D100", # Missing docstring in public module + "D102", # Missing docstring in public method + "D103", # Missing docstring in public function + "D106", # Missing docstring in public nested class + "N801", # Class name should use CapWords convention + "N802", # Function name should be lowercase + "N806", # Variable in function should be lowercase + "N813", # Argument name should be lowercase + "N999", # Invalid module name (e.g. other 'test_xx' modules) + "PLR09", # Various complexity warns: too many args, statements, branches... + "PLR2004", # Magic constants in comparison + "S101", # Use of assert. +] + +[tool.ruff.lint.pydocstyle] +convention = "numpy" + +[tool.setuptools] +zip-safe = false + [tool.setuptools.dynamic] +# NOTE: no dependencies, for now. readme = {file = "README.md", content-type = "text/markdown"} + [tool.setuptools.packages.find] include = ["ncdata*"] where = ["lib"] [tool.setuptools_scm] -version_file = "lib/ncdata/_version.py" local_scheme = "dirty-tag" version_scheme = "release-branch-semver" - - -[tool.black] -line-length = 79 -target-version = ['py310'] -include = '\.pyi?$' - -[tool.isort] -line_length = "79" -profile = "black" -known_local_folder = ["tests"] -known_first_party = "ncdata" -skip = ["lib/ncdata/__init__.py"] +write_to = "lib/ncdata/_version.py" [tool.towncrier] -package = "ncdata" -package_dir = "lib" directory = "docs/changelog_fragments" filename = "docs/change_log.rst" issue_format = "`ISSUE#{issue} `_" +package = "ncdata" +package_dir = "lib" underlines = ["~", "^", "*", "+"] [[tool.towncrier.type]] @@ -113,4 +304,3 @@ showcontent = true directory = "misc" name = "Developer and Internal changes" showcontent = true - diff --git a/tests/__init__.py b/tests/__init__.py index 9d8ecca..d5fe856 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,45 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Tests for :mod:`ncdata`.""" +import os from pathlib import Path +from typing import AnyStr + + +# Note : `_env_bin_path` and the ncgen/ncdump path constants are shared across +# the tests package, so they live here rather than in an individual test module. +def _env_bin_path(exe_name: AnyStr | None = None): + """ + Return a Path object for (an executable in) the environment bin directory. + + Parameters + ---------- + exe_name : str + If set, the name of an executable to append to the path. + + Returns + ------- + exe_path : Path + A path to the bin directory, or an executable file within it. + + Notes + ----- + For use in tests which spawn commands which should call executables within + the Python environment, since many IDEs (Eclipse, PyCharm) don't + automatically include this location in $PATH (as opposed to $PYTHONPATH). + """ + exe_path = Path(os.__file__) + exe_path = (exe_path / "../../../bin").resolve() + if exe_name is not None: + exe_path = exe_path / exe_name + return exe_path + + +NCGEN_PATHSTR = str(_env_bin_path("ncgen")) +NCDUMP_PATHSTR = str(_env_bin_path("ncdump")) testdata_dir = Path(__file__).parent / "testdata" @@ -28,7 +67,8 @@ def __getitem__(self, keys): def version_tuple(version_string): - """Make a comparable tuple from a package version string. + """ + Make a comparable tuple from a package version string. Allows only the final segment to have non-numeric parts. Returns a tuple of the parts. @@ -42,7 +82,5 @@ def version_tuple(version_string): else: parts = parts[:-1] if any(str(int(part)) != part for part in parts): - raise ValueError( - f"invalid numerics in version string: {version_string!r}" - ) + raise ValueError(f"invalid numerics in version string: {version_string!r}") return tuple(int(part) for part in parts) diff --git a/tests/data_testcase_schemas.py b/tests/data_testcase_schemas.py index 8212577..fa7bdaf 100644 --- a/tests/data_testcase_schemas.py +++ b/tests/data_testcase_schemas.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Define a set of "standard" test cases, built as actual netcdf files. @@ -12,7 +16,6 @@ from dataclasses import dataclass from pathlib import Path -from typing import Dict, Union import iris.tests import netCDF4 as nc @@ -48,9 +51,7 @@ def data_types(): # Get the dtype list from the netCDF default fill-values. _nc_dtypes = set(nc.default_fillvals.keys()) # Remove the numpy-only complex types -_nc_dtypes = set( - typename for typename in _nc_dtypes if np.dtype(typename).kind != "c" -) +_nc_dtypes = {typename for typename in _nc_dtypes if np.dtype(typename).kind != "c"} # Also replace 'S1' with our own 'string' type marker _nc_dtypes.remove("S1") _nc_dtypes.add("string") @@ -58,10 +59,10 @@ def data_types(): assert set(data_types()) == _nc_dtypes # Suitable test values for each attribute/data type. -_INT_Approx_2x31 = int(2**31 - 1) -_INT_Approx_2x32 = int(2**32 - 2) -_INT_Approx_2x63 = int(2**63 - 3) -_INT_Approx_2x64 = int(2**64 - 4) +_INT_Approx_2x31 = 2**31 - 1 +_INT_Approx_2x32 = 2**32 - 2 +_INT_Approx_2x63 = 2**63 - 3 +_INT_Approx_2x64 = 2**64 - 4 _Datatype_Sample_Values = { "u1": np.array([3, 250], dtype="u1"), @@ -83,7 +84,7 @@ def data_types(): def _write_nc4_dataset( spec: dict, ds: nc.Dataset, # or an inner Group - parent_spec: dict = None, + parent_spec: dict | None = None, ): """ Create a netcdf test file from a 'testcase spec'. @@ -145,9 +146,7 @@ def objmap(objs): shape = nc_var.shape n_points = int(np.prod(shape)) if nc_var.dtype.kind == "S": - data = np.array( - "abcdefghijklmnopqrstuvwxyz"[:n_points], dtype="S1" - ) + data = np.array("abcdefghijklmnopqrstuvwxyz"[:n_points], dtype="S1") else: data = np.arange(1, n_points + 1) data = data.astype(dtype) @@ -206,9 +205,7 @@ def make_testcase_dataset(filepath, spec): ds.close() -_minimal_variable_test_spec = { - "vars": [dict(name="var_0", dims=[], dtype=int)] -} +_minimal_variable_test_spec = {"vars": [dict(name="var_0", dims=[], dtype=int)]} _simple_test_spec = { @@ -299,7 +296,7 @@ def make_testcase_dataset(filepath, spec): } # Define a sequence of standard testfile specs, with suitable param-names. -_STANDARD_TESTCASES: Dict[str, Union[Path, dict]] = {} +_STANDARD_TESTCASES: dict[str, Path | dict] = {} # A decorator for spec-generating routines. @@ -323,9 +320,7 @@ def _define_simple_testcases(): "ds_Minimal": _minimal_variable_test_spec, "ds_Basic": _simple_test_spec, "ds_testdata1": ( - Path(__file__).parent - / "testdata" - / "toa_brightness_temperature.nc" + Path(__file__).parent / "testdata" / "toa_brightness_temperature.nc" ), "ds_scaleoffset": _scaleoffset_test_spec, "ds_masked_floats": _masked_floats_test_spec, @@ -360,14 +355,11 @@ def _define_iris_testdata_testcases(): param_name = str(filepath) # remove unwanted path elements param_name = param_name.replace(str(_testdirpath), "") - if param_name.endswith(".nc"): - param_name = param_name[:-3] + param_name = param_name.removesuffix(".nc") # replace path-separators and other awkward chars with dunder for char in ("/", ".", "-"): param_name = param_name.replace(char, "__") - # TEMPORARY: skip unstructured ones, just for now, as it makes the run faster if not any(key in param_name for key in EXCLUDES): - # if "small_theta_colpex" in param_name: testcases[f"testdata__{param_name}"] = filepath return testcases @@ -417,9 +409,7 @@ def _define_unit_dtype_testcases(): "attrs": { f"tstatt_type__{dtype_name}__single": _Datatype_Sample_Values[ dtype_name - ][ - 0 - ], + ][0], f"tstatt_type__{dtype_name}__multi": _Datatype_Sample_Values[ dtype_name ], diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3b98f77..19a55c5 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Integration tests for :mod:`ncdata`.""" diff --git a/tests/integration/equivalence_testing_utils.py b/tests/integration/equivalence_testing_utils.py index 521a2a5..6d59299 100644 --- a/tests/integration/equivalence_testing_utils.py +++ b/tests/integration/equivalence_testing_utils.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Utility routines for conversion equivalence testing. @@ -60,12 +64,11 @@ def fix_arrays(a1, a2): """ if a1 is not None and a2 is not None: allmasked_1, allmasked_2 = ( - da.all(da.ma.getmaskarray(arr).compute()) - for arr in (a1, a2) + da.all(da.ma.getmaskarray(arr).compute()) for arr in (a1, a2) ) if allmasked_1 and allmasked_2: - # IN this case == will return 'masked', but for our purposes should be - # considered equal, i.e. 'True' + # IN this case == will return 'masked', but for our purposes + # should be considered equal, i.e. 'True' a1 = np.zeros(a1.shape, dtype=bool) a2 = a1 @@ -81,12 +84,10 @@ def fix_arrays(a1, a2): return a1, a2 # Fix matching of cube data arrays - c1.data, c2.data = fix_arrays( - *(cube.core_data() for cube in (c1, c2)) - ) + c1.data, c2.data = fix_arrays(*(cube.core_data() for cube in (c1, c2))) # Fix matching of all coords points + bounds - for co1, co2 in zip(c1.coords(), c2.coords()): - if isinstance(co1, iris.mesh.MeshCoord): + for co1, co2 in zip(c1.coords(), c2.coords(), strict=False): + if any(isinstance(co, iris.mesh.MeshCoord) for co in (co1, co2)): # Can't write MeshCoords continue co1.points, co2.points = fix_arrays( @@ -112,7 +113,7 @@ def fix_arrays(a1, a2): def set_tiny_chunks(on, size_spec="20Kib"): """Turn on and off the tiny-chunks effect of the 'adjust_chunks' fixture.""" - global _USE_TINY_CHUNKS, _CHUNKSIZE_SPEC + global _USE_TINY_CHUNKS, _CHUNKSIZE_SPEC # noqa: PLW0603 _CHUNKSIZE_SPEC = size_spec _USE_TINY_CHUNKS = on @@ -177,9 +178,9 @@ def nanmask_cube(cube): # Horrible code to list the properties of a netCDF4.Variable object # import inspect +from pathlib import Path import shutil import tempfile -from pathlib import Path import netCDF4 as nc @@ -222,12 +223,12 @@ def prune_cube_varproperties(cube_or_cubes): names = set() for cube in cube_or_cubes: - components = ( - [cube] - + list(cube.coords()) - + list(cube.cell_measures()) - + list(cube.ancillary_variables()) - ) + components = [ + cube, + *list(cube.coords()), + *list(cube.cell_measures()), + *list(cube.ancillary_variables()), + ] for comp in components: names |= prune_attrs_varproperties(comp.attributes) @@ -261,12 +262,12 @@ def remove_cube_nounits(cube_or_cubes): cube_or_cubes = [cube_or_cubes] for cube in cube_or_cubes: - components = ( - [cube] - + list(cube.coords()) - + list(cube.cell_measures()) - + list(cube.ancillary_variables()) - ) + components = [ + cube, + *list(cube.coords()), + *list(cube.cell_measures()), + *list(cube.ancillary_variables()), + ] for comp in components: remove_element_nounits(comp) diff --git a/tests/integration/example_scripts/__init__.py b/tests/integration/example_scripts/__init__.py index cb4b2fa..b0157d9 100644 --- a/tests/integration/example_scripts/__init__.py +++ b/tests/integration/example_scripts/__init__.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Example scripts for :mod:`ncdata`. diff --git a/tests/integration/example_scripts/ex_dataset_print.py b/tests/integration/example_scripts/ex_dataset_print.py index 02fbac8..a5223bc 100644 --- a/tests/integration/example_scripts/ex_dataset_print.py +++ b/tests/integration/example_scripts/ex_dataset_print.py @@ -1,14 +1,18 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Temporary integrational proof-of-concept example for dataset printout.""" import iris -import ncdata.iris as nci from ncdata import NcData, NcDimension, NcVariable +import ncdata.iris as nci from tests import testdata_dir -def sample_printout(): # noqa: D103 +def sample_printout(): iris.FUTURE.datum_support = True filepath = testdata_dir / "toa_brightness_temperature.nc" cubes = iris.load(filepath) diff --git a/tests/integration/example_scripts/ex_iris_saveto_ncdata.py b/tests/integration/example_scripts/ex_iris_saveto_ncdata.py index 361ff06..18c47ee 100644 --- a/tests/integration/example_scripts/ex_iris_saveto_ncdata.py +++ b/tests/integration/example_scripts/ex_iris_saveto_ncdata.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ A proof-of-concept example workflow for :func:`ncdata.iris.from_iris`. @@ -13,7 +17,7 @@ def example_ncdata_from_iris(): """Demonstrate loading from iris and printing the NcData object.""" - print("") + print() print("==============") print("TEMPORARY: iris save-to-ncdata test") iris.FUTURE.datum_support = True diff --git a/tests/integration/example_scripts/ex_iris_xarray_conversion.py b/tests/integration/example_scripts/ex_iris_xarray_conversion.py index 0a193ef..d91ac71 100644 --- a/tests/integration/example_scripts/ex_iris_xarray_conversion.py +++ b/tests/integration/example_scripts/ex_iris_xarray_conversion.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ A proof-of-concept example workflow for :mod:`ncdata.iris_xarray`. @@ -14,7 +18,7 @@ from tests import testdata_dir -def example_from_xr(): # noqa: D103 +def example_from_xr(): filepath = testdata_dir / "toa_brightness_temperature.nc" xrds = xr.open_dataset(filepath, chunks="auto") iris.FUTURE.datum_support = True @@ -47,7 +51,7 @@ def example_from_xr(): # noqa: D103 print("\n") print("============ CONVERT cubes TO xr.Dataset ... =========") - print("") + print() xrds2 = cubes_to_xarray(cubes) print("\nxrds2:\n", xrds2) print("\ntime:\n", xrds2["time"]) @@ -74,10 +78,7 @@ def example_from_xr(): # noqa: D103 print( "xrds2['x'].data is cube.coord('projection_x_coordinate').core_points() : ", - bool( - xrds2["x"].data - is cube.coord("projection_x_coordinate").core_points() - ), + bool(xrds2["x"].data is cube.coord("projection_x_coordinate").core_points()), ) # NOTE: this one does **not** succeed. # TODO: find out exactly why -- ? in some way, because it is a dim coord ? @@ -91,15 +92,9 @@ def example_from_xr(): # noqa: D103 # special care is needed. print( "np.all(xrds2['x'].data == cube.coord('projection_x_coordinate').points) : ", - bool( - np.all( - xrds2["x"].data == cube.coord("projection_x_coordinate").points - ) - ), - ) - assert np.all( - xrds2["x"].data == cube.coord("projection_x_coordinate").points + bool(np.all(xrds2["x"].data == cube.coord("projection_x_coordinate").points)), ) + assert np.all(xrds2["x"].data == cube.coord("projection_x_coordinate").points) if __name__ == "__main__": diff --git a/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py b/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py index 3fde5bf..d8d2c03 100644 --- a/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py +++ b/tests/integration/example_scripts/ex_ncdata_netcdf_conversion.py @@ -1,12 +1,17 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ A proof-of-concept example workflow for :mod:`ncdata.netcdf4`. Showing loading and saving ncdata to/from netcdf files. """ -import tempfile from pathlib import Path from shutil import rmtree +from subprocess import check_call +import tempfile import netCDF4 as nc import numpy as np @@ -18,7 +23,7 @@ from tests import testdata_dir -def example_nc4_load_save_roundtrip(): # noqa: D103 +def example_nc4_load_save_roundtrip(): """Load an existing file ; save to netcdf4 ; check they are the same.""" print("\n----\nNetcdf4 load-save example.") @@ -63,14 +68,14 @@ def example_nc4_save_reload_unlimited_roundtrip(): tempdir_path = Path(tempfile.mkdtemp()) try: - print("") + print() filepath = tempdir_path / "temp_nc_2_unlim.nc" print("Saving to file ", filepath) to_nc4(ncdata, filepath) - import os - print("\n\nCreated output:") - os.system(f"ncdump -h {filepath!s}") + print("\n\nCreated output (ncdump):") + _NCDUMP_EXE_PATH = Path(nc.__file__).parents[4] / "bin" / "ncdump" + check_call([_NCDUMP_EXE_PATH, "-h", filepath]) # noqa: S603 print("\n") ds = nc.Dataset(filepath) @@ -91,8 +96,10 @@ def example_nc4_save_reload_unlimited_roundtrip(): assert list(ds_back.dimensions.keys()) == ["dim_x", "dim_y"] dimx, dimy = [ds_back.dimensions[name] for name in ("dim_x", "dim_y")] - assert dimx.size == len_x and dimx.unlimited - assert dimy.size == len_y and not dimy.unlimited + assert dimx.size == len_x + assert dimx.unlimited + assert dimy.size == len_y + assert not dimy.unlimited assert list(ds_back.variables.keys()) == ["var_x"] varx = ds_back.variables["var_x"] diff --git a/tests/integration/test_exercises.py b/tests/integration/test_exercises.py index 68a33f4..e863b05 100644 --- a/tests/integration/test_exercises.py +++ b/tests/integration/test_exercises.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Temporary : simple tests to exercise the 'ex_' scripts. diff --git a/tests/integration/test_iris_load_and_save_equivalence.py b/tests/integration/test_iris_load_and_save_equivalence.py index 922fe6d..a82f4ea 100644 --- a/tests/integration/test_iris_load_and_save_equivalence.py +++ b/tests/integration/test_iris_load_and_save_equivalence.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Test ncdata.iris by checking roundtrips for standard testcases. @@ -14,7 +18,7 @@ from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences -from tests import version_tuple +from tests import NCDUMP_PATHSTR, version_tuple from tests.data_testcase_schemas import session_testdir, standard_testcase from tests.integration.equivalence_testing_utils import ( adjust_chunks, @@ -48,9 +52,7 @@ def use_irislock(): set_tiny_chunks(_USE_TINY_CHUNKS) -def test_load_direct_vs_viancdata( - standard_testcase, use_irislock, adjust_chunks -): +def test_load_direct_vs_viancdata(standard_testcase, use_irislock, adjust_chunks): specific_excludes = [ # This one has latitude points which exceed the valid_min/max attributes # The netcdf-variable-like transform in Nc4VariableLike._data_array don't @@ -67,8 +69,7 @@ def test_load_direct_vs_viancdata( ] ) if any( - name_fragment in standard_testcase.name - for name_fragment in specific_excludes + name_fragment in standard_testcase.name for name_fragment in specific_excludes ): pytest.skip("excluded testcase") @@ -84,7 +85,7 @@ def test_load_direct_vs_viancdata( print("\nncdata =") print(ncdata) print("\nncdump =") - txt = check_output([f"ncdump {source_filepath}"], shell=True).decode() + txt = check_output([NCDUMP_PATHSTR, str(source_filepath)]).decode() # noqa: S603 print(txt) # Load the testcase with Iris. @@ -99,7 +100,7 @@ def test_load_direct_vs_viancdata( results = [ (c1.name(), cubes_equal__corrected(c1, c2)) - for c1, c2 in zip(iris_cubes, iris_ncdata_cubes) + for c1, c2 in zip(iris_cubes, iris_ncdata_cubes, strict=False) ] expected = [(cube.name(), True) for cube in iris_cubes] result = results == expected @@ -132,8 +133,7 @@ def test_save_direct_vs_viancdata(standard_testcase, tmp_path): # TODO: remove backwards compatibility when we drop Python 3.10 support specific_excludes.append("theta_nodal_xios") if any( - name_fragment in standard_testcase.name - for name_fragment in specific_excludes + name_fragment in standard_testcase.name for name_fragment in specific_excludes ): pytest.skip("excluded testcase") @@ -160,17 +160,13 @@ def test_save_direct_vs_viancdata(standard_testcase, tmp_path): print("\nncdata =") print(ncdata) print("\nncdump ORIGINAL TESTCASE SOURCEFILE =") - txt = check_output([f"ncdump {source_filepath}"], shell=True).decode() + txt = check_output([NCDUMP_PATHSTR, str(source_filepath)]).decode() # noqa: S603 print(txt) print("\nncdump DIRECT FROM IRIS =") - txt = check_output( - [f"ncdump {temp_iris_savepath}"], shell=True - ).decode() + txt = check_output([NCDUMP_PATHSTR, str(temp_iris_savepath)]).decode() # noqa: S603 print(txt) print("\nncdump VIA NCDATA =") - txt = check_output( - [f"ncdump {temp_ncdata_savepath}"], shell=True - ).decode() + txt = check_output([NCDUMP_PATHSTR, str(temp_ncdata_savepath)]).decode() # noqa: S603 print(txt) # Check equivalence diff --git a/tests/integration/test_iris_xarray_roundtrips.py b/tests/integration/test_iris_xarray_roundtrips.py index 7554516..30d1799 100644 --- a/tests/integration/test_iris_xarray_roundtrips.py +++ b/tests/integration/test_iris_xarray_roundtrips.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Test ncdata.iris by checking roundtrips for standard testcases. @@ -22,7 +26,7 @@ from ncdata.utils import dataset_differences from ncdata.xarray import from_xarray -from tests import version_tuple +from tests import NCDUMP_PATHSTR, version_tuple from tests.data_testcase_schemas import ( BAD_LOADSAVE_TESTCASES, session_testdir, @@ -109,7 +113,7 @@ def test_roundtrip_ixi(standard_testcase, use_irislock, adjust_chunks): print("spec =") print(standard_testcase.spec) print("\nncdump =") - txt = check_output([f"ncdump {source_filepath}"], shell=True).decode() + txt = check_output([NCDUMP_PATHSTR, str(source_filepath)]).decode() # noqa: S603 print(txt) # Load the testcase with Iris. @@ -122,9 +126,7 @@ def test_roundtrip_ixi(standard_testcase, use_irislock, adjust_chunks): # _PRUNE_CFVAR_PROPNAMES = False if _PRUNE_CFVAR_PROPNAMES: removed_props = prune_cube_varproperties(iris_cubes) - assert removed_props == set() or removed_props == { - "name", - } + assert removed_props in (set(), {"name"}) # Remove any 'no_units' units, since these do not save correctly # see : https://github.com/SciTools/iris/issues/5368 @@ -158,7 +160,7 @@ def test_roundtrip_ixi(standard_testcase, use_irislock, adjust_chunks): _UNIFY_TIME_UNITS = True # _UNIFY_TIME_UNITS = False if _UNIFY_TIME_UNITS: - for iris_cube, iris_xr_cube in zip(iris_cubes, iris_xr_cubes): + for iris_cube, iris_xr_cube in zip(iris_cubes, iris_xr_cubes, strict=False): for xr_coord in iris_xr_cube.coords(): iris_coords = iris_cube.coords(xr_coord.name()) if len(iris_coords) != 1: @@ -183,7 +185,7 @@ def test_roundtrip_ixi(standard_testcase, use_irislock, adjust_chunks): results = [ (c1.name(), cubes_equal__corrected(c1, c2)) - for c1, c2 in zip(iris_cubes, iris_xr_cubes) + for c1, c2 in zip(iris_cubes, iris_xr_cubes, strict=False) ] expected = [(cube.name(), True) for cube in iris_cubes] result = results == expected @@ -191,9 +193,7 @@ def test_roundtrip_ixi(standard_testcase, use_irislock, adjust_chunks): if not result: # FOR NOW: compare with experimental ncdata comparison. # I know this is a bit circular, but it is useful for debugging, for now ... - result = dataset_differences( - from_iris(iris_cubes), from_iris(iris_xr_cubes) - ) + result = dataset_differences(from_iris(iris_cubes), from_iris(iris_xr_cubes)) assert result == [] @@ -202,9 +202,7 @@ def test_roundtrip_ixi(standard_testcase, use_irislock, adjust_chunks): # It's not currently clear if we *can* find a simple view of what "ought" to work here, # or whether we should be attempting to test this at all. @pytest.mark.skip("Roundtrip testing xarray-iris-xarray : currently disabled.") -def test_roundtrip_xix( - standard_testcase, use_irislock, adjust_chunks, tmp_path -): +def test_roundtrip_xix(standard_testcase, use_irislock, adjust_chunks, tmp_path): source_filepath = standard_testcase.filepath # Skip some cases @@ -235,7 +233,7 @@ def test_roundtrip_xix( print("spec =") print(standard_testcase.spec) print("\nncdump =") - txt = check_output([f"ncdump {source_filepath}"], shell=True).decode() + txt = check_output([NCDUMP_PATHSTR, str(source_filepath)]).decode() # noqa: S603 print(txt) # Load the testcase with Xarray. @@ -303,14 +301,12 @@ def test_roundtrip_xix( mask |= data == fv data = da.ma.masked_array(data, mask=mask) var.data = data - if "calendar" in var.avals: - if var.avals["calendar"] == "gregorian": - # the calendar name 'gregorian' is now deprecated, so Iris replaces it. - var.avals["calendar"] = "standard" - - result = dataset_differences( - ncds_xr, ncds_xr_iris - ) # , check_var_data=False) + if var.avals.get("calendar", "") == "gregorian": + # the calendar name 'gregorian' is now deprecated, + # so Iris replaces it. + var.avals["calendar"] = "standard" + + result = dataset_differences(ncds_xr, ncds_xr_iris) # , check_var_data=False) assert result == [] # TODO: check equivalence, in Xarray terms diff --git a/tests/integration/test_netcdf_roundtrips.py b/tests/integration/test_netcdf_roundtrips.py index d5e136b..ad455a1 100644 --- a/tests/integration/test_netcdf_roundtrips.py +++ b/tests/integration/test_netcdf_roundtrips.py @@ -1,12 +1,15 @@ -""" -Test ncdata.netcdf by checking load-save roundtrips for standard testcases. -""" +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Test ncdata.netcdf by checking load-save roundtrips for standard testcases.""" from subprocess import check_output from ncdata.netcdf4 import from_nc4, to_nc4 from ncdata.utils import dataset_differences +from tests import NCDUMP_PATHSTR from tests.data_testcase_schemas import session_testdir, standard_testcase # Avoid complaints that the imported fixtures are "unused" @@ -34,9 +37,7 @@ def test_basic(standard_testcase, tmp_path): print("\nncdata =") print(ncdata) print("\nncdump =") - txt = check_output( - [f"ncdump {intermediate_filepath}"], shell=True - ).decode() + txt = check_output([NCDUMP_PATHSTR, str(intermediate_filepath)]).decode() # noqa: S603 print(txt) # Check that the re-saved file matches the original diff --git a/tests/integration/test_xarray_load_and_save_equivalence.py b/tests/integration/test_xarray_load_and_save_equivalence.py index aa236db..378fcc1 100644 --- a/tests/integration/test_xarray_load_and_save_equivalence.py +++ b/tests/integration/test_xarray_load_and_save_equivalence.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Test ncdata.xarray by checking roundtrips for standard testcases. @@ -63,9 +67,7 @@ def check_vars_equivalent(v1, v2): assert v1.dims == v2.dims assert v1.dtype == v2.dtype # Numeric compare may need to allow for NaNs : floats *and datetimes* - equal_nan = ( - v1.dtype.kind in "fM" - ) # cannot set kwarg when not applicable + equal_nan = v1.dtype.kind in "fM" # cannot set kwarg when not applicable result = np.array_equal(v1.data, v2.data, equal_nan=equal_nan) if hasattr(result, "compute"): result = result.compute() diff --git a/tests/integration/test_zarr_to_iris.py b/tests/integration/test_zarr_to_iris.py index 9127ce4..b207587 100644 --- a/tests/integration/test_zarr_to_iris.py +++ b/tests/integration/test_zarr_to_iris.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Test conversion of remote and local Zarr store to iris Cube.""" from importlib.resources import files as importlib_files @@ -98,8 +102,7 @@ def _is_url_ok(url): # Check 3 files, as we apparently sometimes get 'partial success' _S3_accessible = all( - _is_url_ok(S3_TEST_PATH + subpath) - for subpath in ["", "/.zattrs", "/.zmetadata"] + _is_url_ok(S3_TEST_PATH + subpath) for subpath in ["", "/.zattrs", "/.zmetadata"] ) @@ -108,7 +111,8 @@ def _is_url_ok(url): ) @pytest.mark.skipif(not _S3_accessible, reason="S3 url not accessible") def test_load_remote_zarr(): - """Test loading a remote Zarr store. + """ + Test loading a remote Zarr store. This is a ~250MB compressed Zarr in an S3 bucket. Conversion is done fully lazily, by passing chunks={} diff --git a/tests/test_samplecode_cdlgen_comparablecdl.py b/tests/test_samplecode_cdlgen_comparablecdl.py index 2245cff..9bd9bc0 100644 --- a/tests/test_samplecode_cdlgen_comparablecdl.py +++ b/tests/test_samplecode_cdlgen_comparablecdl.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ PoC code for netcdf file CDL-based testing mechanisms. @@ -6,52 +10,14 @@ The status and usage of this are yet to be determined. """ -import os import subprocess -from pathlib import Path -from typing import AnyStr, List, Optional import pytest -# Note : `_env_bin_path` and `ncgen_from_cdl` are taken from Iris test code. -# Code here duplicated from Iris v3.4.1 -# https://github.com/SciTools/iris/blob/v3.4.1/lib/iris/tests/stock/netcdf.py#L18-L63 +from tests import NCDUMP_PATHSTR, NCGEN_PATHSTR -def _env_bin_path(exe_name: AnyStr = None): - """ - Return a Path object for (an executable in) the environment bin directory. - - Parameters - ---------- - exe_name : str - If set, the name of an executable to append to the path. - - Returns - ------- - exe_path : Path - A path to the bin directory, or an executable file within it. - - Notes - ----- - For use in tests which spawn commands which should call executables within - the Python environment, since many IDEs (Eclipse, PyCharm) don't - automatically include this location in $PATH (as opposed to $PYTHONPATH). - """ - exe_path = Path(os.__file__) - exe_path = (exe_path / "../../../bin").resolve() - if exe_name is not None: - exe_path = exe_path / exe_name - return exe_path - - -NCGEN_PATHSTR = str(_env_bin_path("ncgen")) -NCDUMP_PATHSTR = str(_env_bin_path("ncdump")) - - -def ncgen_from_cdl( - cdl_str: Optional[str], cdl_path: Optional[str], nc_path: str -): +def ncgen_from_cdl(cdl_str: str | None, cdl_path: str | None, nc_path: str): """ Generate a test netcdf file from cdl. @@ -76,7 +42,7 @@ def ncgen_from_cdl( """ if cdl_str and cdl_path: - with open(cdl_path, "w") as f_out: + with cdl_path.open("w") as f_out: f_out.write(cdl_str) if cdl_path: # Create netcdf from stored CDL file. @@ -89,7 +55,7 @@ def ncgen_from_cdl( call_args = [NCGEN_PATHSTR, "-k3", "-o", nc_path] call_kwargs = dict(input=cdl_str, encoding="ascii") - subprocess.run(call_args, check=True, **call_kwargs) + subprocess.run(call_args, check=True, **call_kwargs) # noqa: S603 # CDL to create a reference file with "all" features included. @@ -137,7 +103,7 @@ def ncgen_from_cdl( """ -def comparable_cdl(text: str) -> List[str]: +def comparable_cdl(text: str) -> list[str]: """ Convert a CDL string to a list of stripped lines, with certain problematic things removed. The resulting list of strings should be comparable between identical @@ -157,7 +123,8 @@ def comparable_cdl(text: str) -> List[str]: ] # Also exclude the first line, which includes the filename hdr, lines = lines[0], lines[1:] - assert hdr.startswith("netcdf") and hdr[-1] == "{" + assert hdr.startswith("netcdf") + assert hdr[-1] == "{" return lines @@ -167,9 +134,9 @@ def testdata_cdl(tmp_path_factory): cdl_path = tmpdir_path / "tmp_nccompare_test.cdl" nc_path = tmpdir_path / "tmp_nccompare_test.nc" ncgen_from_cdl(cdl_str=_base_cdl, cdl_path=cdl_path, nc_path=nc_path) - bytes = subprocess.check_output([NCDUMP_PATHSTR, "-h", nc_path]) - cdl_regen_text = bytes.decode() - return cdl_regen_text + cdl_bytes = subprocess.check_output([NCDUMP_PATHSTR, "-h", nc_path]) # noqa: S603 + cdl_text = cdl_bytes.decode() + return cdl_text def test_ncgen_from_cdl(testdata_cdl): diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 358a9a2..1fb04e5 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Unit tests for :mod:`ncdata`.""" diff --git a/tests/unit/core/__init__.py b/tests/unit/core/__init__.py index 4269eb7..dbf4412 100644 --- a/tests/unit/core/__init__.py +++ b/tests/unit/core/__init__.py @@ -1 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Unit tests for :mod:`ncdata.core`.""" diff --git a/tests/unit/core/test_AttributeAccessMixin.py b/tests/unit/core/test_AttributeAccessMixin.py index c3f243f..148bead 100644 --- a/tests/unit/core/test_AttributeAccessMixin.py +++ b/tests/unit/core/test_AttributeAccessMixin.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Tests for class :class:`ncdata._core._AttributeAccessMixin`. @@ -26,7 +30,7 @@ def test_gettattr(self, sample_object): def test_getattr_absent(self, sample_object): # Check that fetching a non-existent attribute returns None. - sample_object.get_attrval("q") is None + assert sample_object.get_attrval("q") is None def test_setattr(self, sample_object): content = np.array([1, 2]) diff --git a/tests/unit/core/test_AttrvalsDict.py b/tests/unit/core/test_AttrvalsDict.py index 7a92911..0084153 100644 --- a/tests/unit/core/test_AttrvalsDict.py +++ b/tests/unit/core/test_AttrvalsDict.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Tests for class :class:`ncdata._core.AttrvalsDict`.""" from copy import deepcopy @@ -36,16 +40,16 @@ def setup(self): @pytest.fixture(params=_alltype_opts) def itemtype(request): - yield request.param + return request.param @pytest.fixture(params=_alltype_opts) def value_oftype(request): - yield _testitem[request.param] + return _testitem[request.param] class Test_setitem(MixinEquivTest): - """Testing equivalence to ".attributes" """ + """Testing equivalence to ".attributes".""" def test_types(self, value_oftype): a, b = self.vars_a_b @@ -79,7 +83,7 @@ def test_preexist(self): class Test_getitems(MixinEquivTest): - """Testing equivalence to ".attributes" """ + """Testing equivalence to ".attributes".""" def test_getitem(self, value_oftype): v = self.var @@ -164,9 +168,7 @@ def test_del(self): def test_update(self): v = self.var # renaming : name of NcAttribute supplied as a value is lost - v.avals.update( - {"q": 49.3, "ZZ": NcAttribute("", [1, 2]), "a": 17} - ) + v.avals.update({"q": 49.3, "ZZ": NcAttribute("", [1, 2]), "a": 17}) # ordering : "q" and "ZZ" are added after, but "a" stays where it was assert list(v.attributes.keys()) == ["a", "b", "q", "ZZ"] assert v.attributes["a"] == NcAttribute("a", 17) diff --git a/tests/unit/core/test_NameMap.py b/tests/unit/core/test_NameMap.py index 67e22b9..43310aa 100644 --- a/tests/unit/core/test_NameMap.py +++ b/tests/unit/core/test_NameMap.py @@ -1,6 +1,8 @@ -""" -Tests for class :class:`ncdata.NameMap`. -""" +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Tests for class :class:`ncdata.NameMap`.""" from copy import deepcopy @@ -61,7 +63,7 @@ def test_item_pairs(self, item_type_and_kwargs): assert namemap.item_type == item_type assert len(namemap) == len(_test_items) assert list(namemap.keys()) == [item.name for item in _test_items] - assert all(namemap[key].name == key for key in namemap.keys()) + assert all(namemap[key].name == key for key in namemap) def test_nameless_items(self): """Check we can construct like a "normal" dictionary when item_type is None.""" @@ -88,18 +90,18 @@ def test_list_arg(self, item_type_and_kwargs): assert namemap.item_type == item_type assert len(namemap) == len(_test_items) assert list(namemap.keys()) == [item.name for item in _test_items] - assert all(namemap[key].name == key for key in namemap.keys()) + assert all(namemap[key].name == key for key in namemap) def test_map_arg(self, item_type_and_kwargs): item_type, kwargs = item_type_and_kwargs # Construct a dictionary, with *arbitrary* keys (they are ignored). - arg = {count: item for count, item in enumerate(_test_items)} + arg = dict(enumerate(_test_items)) namemap = NameMap.from_items(arg, **kwargs) assert isinstance(namemap, NameMap) assert namemap.item_type == item_type assert len(namemap) == len(_test_items) assert list(namemap.keys()) == [item.name for item in _test_items] - assert all(namemap[key].name == key for key in namemap.keys()) + assert all(namemap[key].name == key for key in namemap) def test_items_typemismatch(self, item_type_and_kwargs): item_type, kwargs = item_type_and_kwargs @@ -221,13 +223,12 @@ def test_attributes__namevaluemap(self, target_itemtype, multiple): vals = [val.value for val in vals] assert vals == list(arg.values()) - @pytest.mark.parametrize( - "arg", [[], {}, None], ids=["list", "map", "none"] - ) + @pytest.mark.parametrize("arg", [[], {}, None], ids=["list", "map", "none"]) def test_attributes_empty(self, arg): # Just check correct construction from empty args. namemap = NameMap.from_items(arg, item_type=NcAttribute) - assert namemap == {} and namemap is not arg + assert namemap == {} + assert namemap is not arg class Test_copy: @@ -239,7 +240,7 @@ def test_copy(self, item_type): assert result.item_type == item_type # NOTE: not a *deep* copy : contains the same item objects assert all( - e1 is e2 for e1, e2 in zip(result.values(), source.values()) + e1 is e2 for e1, e2 in zip(result.values(), source.values(), strict=True) ) @@ -248,7 +249,7 @@ def test_sametyped_item(self, item_type): source = sample_namemap(item_type=item_type) testmap = source.copy() testmap.add(NamedItem("qq")) - assert set(testmap) == set(["qq"]) | set(source) + assert set(testmap) == {"qq"} | set(source) def test_differenttype_item(self, item_type): testmap = sample_namemap(item_type=item_type) @@ -288,7 +289,7 @@ def test_overwrites(self): testmap.addall(new_items) assert list(testmap.keys()) == ["a", "b", "c", "d", "x"] assert set(testmap.values()) == ( - (set(source.values()) - set([source["b"]])) | set(new_items) + (set(source.values()) - {source["b"]}) | set(new_items) ) diff --git a/tests/unit/core/test_NcAttribute.py b/tests/unit/core/test_NcAttribute.py index 064b29e..9427c91 100644 --- a/tests/unit/core/test_NcAttribute.py +++ b/tests/unit/core/test_NcAttribute.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Tests for class :class:`ncdata.NcAttribute`. @@ -54,9 +58,7 @@ def structuretype(request): def attrvalue(datatype, structuretype): # get basic type = 'int', 'float', 'string' - (datatype,) = ( - key for key in _attribute_testdata.keys() if key in datatype - ) + (datatype,) = (key for key in _attribute_testdata if key in datatype) # fetch a set of values values = _attribute_testdata[datatype] @@ -107,14 +109,13 @@ def test_value(self, datatype, structuretype): assert all(isinstance(x, str) for x in result) else: assert isinstance(result, str) + # Numeric data is returned as arrays or array scalars. + elif is_vector: + assert isinstance(result, np.ndarray) + assert result.ndim == 1 else: - # Numeric data is returned as arrays or array scalars. - if is_vector: - assert isinstance(result, np.ndarray) - assert result.ndim == 1 - else: - # scalar : result is *always* an array scalar (e.g. np.float64) - assert result.shape == () + # scalar : result is *always* an array scalar (e.g. np.float64) + assert result.shape == () class Test_NcAttribute__str_repr: @@ -165,11 +166,9 @@ def test_value(self, datatype, structuretype): assert result == attr assert result.name == attr.name assert result.value is not attr.value - assert ( - result.value.dtype == attr.value.dtype - and result.value.shape == attr.value.shape - and np.all(result.value == attr.value) - ) + assert result.value.dtype == attr.value.dtype + assert result.value.shape == attr.value.shape + assert np.all(result.value == attr.value) class Test_NcAttribute__eq__: @@ -219,7 +218,8 @@ def test_eq_onestring_arrayofonestring(self): assert attr1 == attr2 assert attr1.value.dtype == " NcData: def test_target_types(targettype, tmp_path): """Check the various ways of specifying the output location.""" # Testcase is minimal spec with just one scalar variable. - test_spec = { - "vars": [dict(name="x", dims=[], dtype=np.float32, data=[1.23])] - } + test_spec = {"vars": [dict(name="x", dims=[], dtype=np.float32, data=[1.23])]} original_path = tmp_path / "input.nc" ncdata = file_and_ncdata_from_spec(original_path, test_spec) @@ -66,7 +67,7 @@ def test_target_types(targettype, tmp_path): assert dataset_differences(target_path, original_path) == [] -def fetch_nc_var(nc_file: nc.Dataset, var_path: str or List[str]): +def fetch_nc_var(nc_file: nc.Dataset, var_path: str or list[str]): """ Return a variable instance from an open dataset for inspection. @@ -108,9 +109,7 @@ def test_var_kwargs(tmp_path): "groups": [ { "name": "sub_subgroup_1_2", - "vars": [ - dict(name="var_xx", dims=["x"], dtype=np.float32) - ], + "vars": [dict(name="var_xx", dims=["x"], dtype=np.float32)], }, ], }, @@ -143,9 +142,7 @@ def test_var_kwargs(tmp_path): assert var_2.filters()["zlib"] is False assert var_2.chunking() == "contiguous" - var_3 = fetch_nc_var( - saved_ds, ["subgroup_1", "sub_subgroup_1_2", "var_xx"] - ) + var_3 = fetch_nc_var(saved_ds, ["subgroup_1", "sub_subgroup_1_2", "var_xx"]) assert var_3.filters()["zlib"] is False assert var_3.chunking() == [2] @@ -153,9 +150,7 @@ def test_var_kwargs(tmp_path): def test_var_kwargs__bad_kwarg(tmp_path): """Check that unsuitable var_kwargs entries are errored.""" # Testcase is a minimal spec with just one scalar variable. - test_spec = { - "vars": [dict(name="x", dims=[], dtype=np.float32, data=[1.23])] - } + test_spec = {"vars": [dict(name="x", dims=[], dtype=np.float32, data=[1.23])]} original_path = tmp_path / "input.nc" ncdata = file_and_ncdata_from_spec(original_path, test_spec) diff --git a/tests/unit/tests/__init__.py b/tests/unit/tests/__init__.py new file mode 100644 index 0000000..efb8edb --- /dev/null +++ b/tests/unit/tests/__init__.py @@ -0,0 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Unit tests for :mod:`tests`.""" diff --git a/tests/unit/tests/test_version_tuple.py b/tests/unit/tests/test_version_tuple.py index dabdbec..6068ce6 100644 --- a/tests/unit/tests/test_version_tuple.py +++ b/tests/unit/tests/test_version_tuple.py @@ -1,6 +1,8 @@ -""" -Tests for the test utility :func:`tests.version_majorminor`. -""" +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Tests for the test utility :func:`tests.version_majorminor`.""" import pytest diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py index 7e82563..2a13dd9 100644 --- a/tests/unit/utils/__init__.py +++ b/tests/unit/utils/__init__.py @@ -1 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Unit tests for :mod:`ncdata.utils`.""" diff --git a/tests/unit/utils/compare_nc_datasets/__init__.py b/tests/unit/utils/compare_nc_datasets/__init__.py index 7f699aa..3b7b71c 100644 --- a/tests/unit/utils/compare_nc_datasets/__init__.py +++ b/tests/unit/utils/compare_nc_datasets/__init__.py @@ -1 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Unit tests for :mod:`ncdata.utils._compare_nc_datasets`.""" diff --git a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py index 9969a13..d965c85 100644 --- a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py +++ b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__additional.py @@ -1,10 +1,14 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Tests for :mod:`tests.unit.netcdf._compare_nc_files` Split in two files ... * HERE: "additional" tests cover subsidiary routines and the main API usage modes. * ( ALSO: "mainfunctions" (q.v.) cover the core functionality - -- which elements are compared and what errors this constructs. ) + -- which elements are compared and what errors this constructs. ). """ import shutil @@ -54,14 +58,10 @@ def test_diff(self): def test_difforder(self): errs = _namelist_differences(["a", "b"], ["b", "a"], "named-elements") - assert errs == [ - "named-elements do not match: ['a', 'b'] != ['b', 'a']" - ] + assert errs == ["named-elements do not match: ['a', 'b'] != ['b', 'a']"] def test_difforder_tolerant_warns(self): - with pytest.warns( - UserWarning, match="Ignoring: named-elements do not match" - ): + with pytest.warns(UserWarning, match="Ignoring: named-elements do not match"): errs = _namelist_differences( ["a", "b"], ["b", "a"], @@ -167,9 +167,7 @@ def test_compare_attributes_values__data_mismatch(self): obj1 = self.Nc4ObjectWithAttrsMimic(a=1, b=2, c=3) obj2 = self.Nc4ObjectWithAttrsMimic(a=1, b=-77, c=3) errs = _attribute_differences(obj1, obj2, "") - assert errs == [ - ' "b" attribute values differ : 2 != -77' - ] + assert errs == [' "b" attribute values differ : 2 != -77'] def test_compare_attributes_values__dtype_mismatch__length(self): # Attributes of different dtypes, even though values == @@ -225,8 +223,10 @@ def test_compare_attributes_values__dtype_and_data_mismatch(self): obj2 = self.Nc4ObjectWithAttrsMimic(a=np.float64(1)) errs = _attribute_differences(obj1, obj2, "") assert errs == [ - ' "a" attribute datatypes differ : ' - "dtype('float32') != dtype('float64')" + ( + ' "a" attribute datatypes differ : ' + "dtype('float32') != dtype('float64')" + ) ] def test_compare_attributes_values__data_arrays_match(self): @@ -257,10 +257,7 @@ def test_compare_attributes_values__data_arrays_shape_mismatch(self): obj2 = self.Nc4ObjectWithAttrsMimic(a=array[:-1]) errs = _attribute_differences(obj1, obj2, "") assert errs == [ - ( - ' "a" attribute values differ : ' - "[0, 1, 2] != [0, 1]" - ) + (' "a" attribute values differ : [0, 1, 2] != [0, 1]') ] def test_compare_attributes_values__data_arrays_value_mismatch(self): @@ -305,8 +302,10 @@ def test_compare_attributes_values__string_nonstring(self): obj2 = self.Nc4ObjectWithAttrsMimic(a="1") errs = _attribute_differences(obj1, obj2, "") assert errs == [ - ' "a" attribute datatypes differ : ' - "dtype('int64') != " + ( + ' "a" attribute datatypes differ : ' + "dtype('int64') != " + ) ] def test_compare_attributes_values__string_match(self): @@ -338,8 +337,10 @@ def test_compare_attributes_values__string_array_mismatch(self): obj2 = self.Nc4ObjectWithAttrsMimic(S=["a", "c"]) errs = _attribute_differences(obj1, obj2, "") assert errs == [ - ' "S" attribute values differ : ' - "['a', 'b'] != ['a', 'c']" + ( + ' "S" attribute values differ : ' + "['a', 'b'] != ['a', 'c']" + ) ] def test_compare_attributes__ncdata_string_scalar_array(self): @@ -408,9 +409,7 @@ def test_identical_datasets(self, samefiles_filesonly, sourcetype): if ds: ds.close() - def test_small_difference( - self, samefiles_bothtypes, temp_ncfiles_dir, sourcetype - ): + def test_small_difference(self, samefiles_bothtypes, temp_ncfiles_dir, sourcetype): source1, source2 = samefiles_bothtypes if sourcetype == "InputsFile": diff --git a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py index 482f794..60e674e 100644 --- a/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py +++ b/tests/unit/utils/compare_nc_datasets/test_dataset_differences__mainfunctions.py @@ -1,10 +1,14 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Tests for :mod:`tests.unit.netcdf._compare_nc_files` Split in two files ... * HERE: "mainfunctions" cover the core functionality -- which elements are compared and what errors this constructs. * ( ALSO: "additional" tests (q.v.) cover subsidiary routines and the - main API usage modes. ) + main API usage modes. ). """ import numpy as np @@ -17,12 +21,10 @@ # data_types # avoid 'unused' warning -@pytest.fixture( - params=["in_named", "in_unnamed", "in_namedgroup", "in_unnamedgroup"] -) +@pytest.fixture(params=["in_named", "in_unnamed", "in_namedgroup", "in_unnamedgroup"]) def group_context(request): """ - The different contexts of locations in a dataset + The different contexts of locations in a dataset. In which an element (dimension, group or variable) might be found, and which might appear different in the mismatch-error messages. @@ -120,9 +122,7 @@ def dimension_testdata(self, group_context): @pytest.fixture(autouse=True) def _dims_data(self, group_context): - data1, data2 = [ - self.dimension_testdata(group_context) for _ in range(2) - ] + data1, data2 = [self.dimension_testdata(group_context) for _ in range(2)] location = data2 if "group" in group_context: location = location.groups["inner_group"] @@ -136,8 +136,10 @@ def test_name(self): self.dims.rename("x", "q") errs = dataset_differences(self.data1, self.data2) expected = [ - f"{self.location_string} dimension lists do not match: " - "['x', 'y'] != ['q', 'y']" + ( + f"{self.location_string} dimension lists do not match: " + "['x', 'y'] != ['q', 'y']" + ) ] check(errs, expected) @@ -151,9 +153,7 @@ def test_size(self): ] check(errs, expected) - @pytest.mark.parametrize( - "check_unlim", ["unlims_checked", "unlims_unchecked"] - ) + @pytest.mark.parametrize("check_unlim", ["unlims_checked", "unlims_unchecked"]) def test_unlimited(self, check_unlim): self.dims["y"].unlimited = True @@ -166,8 +166,11 @@ def test_unlimited(self, check_unlim): if do_check_unlims: expected = [ - f'{self.location_string} "y" dimension has different "unlimited" status : ' - "False != True" + ( + f'{self.location_string} "y" dimension ' + 'has different "unlimited" status : ' + "False != True" + ) ] else: expected = [] @@ -186,8 +189,10 @@ def test_ordering(self, order_checking): if do_ordercheck: expected = [ - f"{self.location_string} dimension lists do not match: " - "['x', 'y'] != ['y', 'x']" + ( + f"{self.location_string} dimension lists do not match: " + "['x', 'y'] != ['y', 'x']" + ) ] else: expected = [] @@ -203,8 +208,7 @@ def test_extra_or_missing(self): errs = dataset_differences(self.data1, self.data2) expected = [ - f"{self.location_string} dimension lists do not match: " - "['x', 'y'] != ['x']" + f"{self.location_string} dimension lists do not match: ['x', 'y'] != ['x']" ] check(errs, expected) @@ -234,9 +238,7 @@ def attribute_testdata(self, group_context): @pytest.fixture(autouse=True) def _attrs_data(self, group_context, attr_context): - data1, data2 = [ - self.attribute_testdata(group_context) for _ in range(2) - ] + data1, data2 = [self.attribute_testdata(group_context) for _ in range(2)] location = data2 if "group" in group_context: location = location.groups["inner_group"] @@ -255,8 +257,10 @@ def test_name(self): errs = dataset_differences(self.data1, self.data2) expected = [ - f"{self.location_string} attribute lists do not match: " - "['att1', 'att2'] != ['changed', 'att2']" + ( + f"{self.location_string} attribute lists do not match: " + "['att1', 'att2'] != ['changed', 'att2']" + ) ] check(errs, expected) @@ -265,13 +269,12 @@ def test_value(self, attr_context): errs = dataset_differences(self.data1, self.data2) - if "variable" in attr_context: - value_string = "1" - else: - value_string = "11" + value_string = "1" if "variable" in attr_context else "11" expected = [ - f'{self.location_string} "att1" attribute values differ : ' - f"{value_string} != 999" + ( + f'{self.location_string} "att1" attribute values differ : ' + f"{value_string} != 999" + ) ] check(errs, expected) @@ -287,8 +290,10 @@ def test_ordering(self, order_checking): if do_ordercheck: expected = [ - f"{self.location_string} attribute lists do not match: " - "['att1', 'att2'] != ['att2', 'att1']" + ( + f"{self.location_string} attribute lists do not match: " + "['att1', 'att2'] != ['att2', 'att1']" + ) ] else: expected = [] @@ -303,8 +308,10 @@ def test_extra_or_missing(self, order_checking): ) expected = [ - f"{self.location_string} attribute lists do not match: " - "['att1', 'att2'] != ['att2']" + ( + f"{self.location_string} attribute lists do not match: " + "['att1', 'att2'] != ['att2']" + ) ] check(errs, expected) @@ -316,9 +323,7 @@ def test_fillvalue_anyorder(self, attname): attr_pair = [NcAttribute(name, 1), NcAttribute("x", 1)] data1, data2 = [ NcData( - variables=[ - NcVariable("vx", (), data=np.array(0.0), attributes=attrs) - ] + variables=[NcVariable("vx", (), data=np.array(0.0), attributes=attrs)] ) for attrs in (attr_pair, attr_pair[::-1]) ] @@ -327,8 +332,10 @@ def test_fillvalue_anyorder(self, attname): if "generic" in attname: expected = [ - 'Dataset variable "vx" attribute lists do not match: ' - "['anyold', 'x'] != ['x', 'anyold']" + ( + 'Dataset variable "vx" attribute lists do not match: ' + "['anyold', 'x'] != ['x', 'anyold']" + ) ] else: expected = [] @@ -380,8 +387,10 @@ def test_var_names(self): errs = dataset_differences(self.data1, self.data2) expected = [ - f"{self.location_string} variable lists do not match: " - "['v1', 'v2'] != ['v1', 'q']" + ( + f"{self.location_string} variable lists do not match: " + "['v1', 'v2'] != ['v1', 'q']" + ) ] check(errs, expected) @@ -397,8 +406,10 @@ def test_var_order(self, order_checking): if do_ordercheck: expected = [ - f"{self.location_string} variable lists do not match: " - "['v1', 'v2'] != ['v2', 'v1']" + ( + f"{self.location_string} variable lists do not match: " + "['v1', 'v2'] != ['v2', 'v1']" + ) ] else: expected = [] @@ -413,8 +424,10 @@ def test_vars_extra_or_missing(self, order_checking): ) expected = [ - f"{self.location_string} variable lists do not match: " - "['v1', 'v2'] != ['v2']" + ( + f"{self.location_string} variable lists do not match: " + "['v1', 'v2'] != ['v2']" + ) ] check(errs, expected) @@ -441,9 +454,7 @@ def test_group_names(self): errs = dataset_differences(self.data1, self.data2) - expected = [ - "Dataset subgroup lists do not match: ['g1', 'g2'] != ['g1', 'q']" - ] + expected = ["Dataset subgroup lists do not match: ['g1', 'g2'] != ['g1', 'q']"] check(errs, expected) def test_group_order(self, order_checking): @@ -458,8 +469,7 @@ def test_group_order(self, order_checking): if do_ordercheck: expected = [ - "Dataset subgroup lists do not match: " - "['g1', 'g2'] != ['g2', 'g1']" + "Dataset subgroup lists do not match: ['g1', 'g2'] != ['g2', 'g1']" ] else: expected = [] @@ -474,7 +484,5 @@ def test_groups_extra_or_missing(self, order_checking): ) # NB since the sets are different, the ordering control has no effect - expected = [ - "Dataset subgroup lists do not match: ['g1', 'g2'] != ['g2']" - ] + expected = ["Dataset subgroup lists do not match: ['g1', 'g2'] != ['g2']"] check(errs, expected) diff --git a/tests/unit/utils/compare_nc_datasets/test_variable_differences.py b/tests/unit/utils/compare_nc_datasets/test_variable_differences.py index 2c7e55b..84e8581 100644 --- a/tests/unit/utils/compare_nc_datasets/test_variable_differences.py +++ b/tests/unit/utils/compare_nc_datasets/test_variable_differences.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. import dask.array as da import numpy as np import pytest @@ -21,15 +25,14 @@ class TestSimpleProperties: @pytest.fixture(autouse=True) def _vars_data(self): self.var1, self.var2 = [ - NcVariable("v1", ("y", "x"), data=np.zeros((2, 3))) - for _ in range(2) + NcVariable("v1", ("y", "x"), data=np.zeros((2, 3))) for _ in range(2) ] def test_var_names(self): self.var2.name = "q" errs = variable_differences(self.var1, self.var2) - expected = ['Variable "v1 / q" names differ : ' "'v1' != 'q'"] + expected = ["Variable \"v1 / q\" names differ : 'v1' != 'q'"] check(errs, expected) def test_var_dims__reorder(self): @@ -41,9 +44,7 @@ def test_var_dims__reorder(self): errs = variable_differences(self.var1, self.var2) - expected = [ - 'Variable "v1" dimensions differ : ' "('y', 'x') != ('x', 'y')" - ] + expected = ["Variable \"v1\" dimensions differ : ('y', 'x') != ('x', 'y')"] check(errs, expected) def test_var_dims__extra_or_missing(self): @@ -77,8 +78,7 @@ def test_numbers_v_strings(self): # Test the comparison errs = variable_differences(self.var1, self.var2) expected = [ - 'Variable "v1" datatypes differ : ' - "dtype('float64') != dtype('S5')" + "Variable \"v1\" datatypes differ : dtype('float64') != dtype('S5')" ] check(errs, expected) @@ -97,8 +97,7 @@ def test_ints_v_floats(self, equaldata): errs = variable_differences(self.var1, self.var2) expected = [ - 'Variable "v1" datatypes differ : ' - "dtype('float64') != dtype('int32')" + "Variable \"v1\" datatypes differ : dtype('float64') != dtype('int32')" ] if not equaldata: expected.append( @@ -123,8 +122,7 @@ def test_wordlengths(self, equaldata): errs = variable_differences(self.var1, self.var2) expected = [ - 'Variable "v1" datatypes differ : ' - "dtype('float64') != dtype('float32')" + "Variable \"v1\" datatypes differ : dtype('float64') != dtype('float32')" ] if not equaldata: expected.append( @@ -153,8 +151,7 @@ def test_signed_unsigned(self, equaldata): errs = variable_differences(self.var1, self.var2) expected = [ - 'Variable "v1" datatypes differ : ' - "dtype('int64') != dtype('uint64')" + "Variable \"v1\" datatypes differ : dtype('int64') != dtype('uint64')" ] if not equaldata: expected.append( @@ -165,8 +162,8 @@ def test_signed_unsigned(self, equaldata): @pytest.mark.parametrize("given", ["nodata", "data", "dtype"]) def test_nodata_nodtype(self, given): - # Check that we can correctly compare a variable with NO specified data or dtype, - # with one that may have either. + # Check that we can correctly compare a variable with NO specified data + # or dtype, with one that may have either. # N.B. this omits comparing 2 variables with dtype only. See following test, # 'test_nodata_withdtype` for that. v1 = NcVariable("x") @@ -180,9 +177,7 @@ def test_nodata_nodtype(self, given): ] elif given == "dtype": kwargs["dtype"] = np.float32 - expected = [ - "Variable \"x\" datatypes differ : None != dtype('float32')" - ] + expected = ["Variable \"x\" datatypes differ : None != dtype('float32')"] elif given == "nodata": expected = [] else: @@ -225,9 +220,7 @@ def test_differences_ignore_datacheck(self, differs): nodata = differs.endswith("nodata") if differs.startswith("dims"): dims2 = ["x"] - expected = [ - "Variable \"x\" dimensions differ : ('x', 'y') != ('x',)" - ] + expected = ["Variable \"x\" dimensions differ : ('x', 'y') != ('x',)"] if nodata: # In this case, either one or both have no data. shape1 = None @@ -236,9 +229,7 @@ def test_differences_ignore_datacheck(self, differs): shape2 = None else: # Only one has data --> triggers a shape warning too. - expected.append( - 'Variable "x" shapes differ : None != (2, 3)' - ) + expected.append('Variable "x" shapes differ : None != (2, 3)') elif differs.startswith("shapes"): if nodata: # In this case one has data (and therefore a shape), and the other not. @@ -278,9 +269,7 @@ def test_no_values_check(self): def test_print_bad_nprint(self): msg = "'show_n_diffs' must be >=1 : got 0." with pytest.raises(ValueError, match=msg): - variable_differences( - self.var1, self.var2, show_n_first_different=0 - ) + variable_differences(self.var1, self.var2, show_n_first_different=0) @pytest.mark.parametrize("ndiffs", [1, 2, 3]) def test_ndiffs(self, ndiffs): @@ -289,31 +278,24 @@ def test_ndiffs(self, ndiffs): detail = { 1: "[(0, 1)] : LHS=[1.0], RHS=[2.0]", 2: "[(0, 1), (0, 2)] : LHS=[1.0, 2.0], RHS=[2.0, 3.0]", - 3: ( - "[(0, 1), (0, 2), ...] : " - "LHS=[1.0, 2.0, ...], RHS=[2.0, 3.0, ...]" - ), + 3: ("[(0, 1), (0, 2), ...] : LHS=[1.0, 2.0, ...], RHS=[2.0, 3.0, ...]"), }[ndiffs] expected = [ - f'Variable "v1" data contents differ, at {ndiffs} points: ' - f"@INDICES{detail}" + f'Variable "v1" data contents differ, at {ndiffs} points: @INDICES{detail}' ] check(errs, expected) @pytest.mark.parametrize("nprint", [1, 2, 3]) def test_show_n_first_different(self, nprint): self.var2.data.flat[1:3] += 1 - errs = variable_differences( - self.var1, self.var2, show_n_first_different=nprint - ) + errs = variable_differences(self.var1, self.var2, show_n_first_different=nprint) detail = { 1: "[(0, 1), ...] : LHS=[1.0, ...], RHS=[2.0, ...]", 2: "[(0, 1), (0, 2)] : LHS=[1.0, 2.0], RHS=[2.0, 3.0]", 3: "[(0, 1), (0, 2)] : LHS=[1.0, 2.0], RHS=[2.0, 3.0]", }[nprint] expected = [ - f'Variable "v1" data contents differ, at 2 points: ' - f"@INDICES{detail}" + f'Variable "v1" data contents differ, at 2 points: @INDICES{detail}' ] check(errs, expected) @@ -346,8 +328,10 @@ def test_masked(self, datavalues, masks): expected = [] else: expected = [ - 'Variable "v1" data contents differ, at 1 points: ' - "@INDICES[(1,)] : LHS=[1.0], RHS=[masked]" + ( + 'Variable "v1" data contents differ, at 1 points: ' + "@INDICES[(1,)] : LHS=[1.0], RHS=[masked]" + ) ] check(errs, expected) @@ -362,8 +346,10 @@ def test_nans(self, nans): expected = [] else: expected = [ - 'Variable "v1" data contents differ, at 1 points: ' - "@INDICES[(1,)] : LHS=[1.0], RHS=[nan]" + ( + 'Variable "v1" data contents differ, at 1 points: ' + "@INDICES[(1,)] : LHS=[1.0], RHS=[nan]" + ) ] check(errs, expected) @@ -374,28 +360,30 @@ def test_scalar(self): var.data = np.array(value, dtype=var.dtype) errs = variable_differences(self.var1, self.var2) expected = [ - 'Variable "v1" data contents differ, at 1 points: ' - "@INDICES[(0,)] : LHS=[0.0], RHS=[1.0]" + ( + 'Variable "v1" data contents differ, at 1 points: ' + "@INDICES[(0,)] : LHS=[0.0], RHS=[1.0]" + ) ] check(errs, expected) - @pytest.mark.parametrize( - "argtypes", ["real_real", "real_lazy", "lazy_lazy"] - ) + @pytest.mark.parametrize("argtypes", ["real_real", "real_lazy", "lazy_lazy"]) def test_real_and_lazy(self, argtypes): type1, type2 = argtypes[:4], argtypes[-4:] # fix the testvar to create a difference self.var2.data[1:2] += 1 # setup vars with lazy/real data arrays - for arraytype, var in zip([type1, type2], [self.var1, self.var2]): + for arraytype, var in [(type1, self.var1), (type2, self.var2)]: if arraytype == "lazy": var.data = da.from_array(var.data, chunks=-1) # compare + check results errs = variable_differences(self.var1, self.var2) # N.B. the result should be the same in all cases expected = [ - 'Variable "v1" data contents differ, at 1 points: ' - "@INDICES[(1,)] : LHS=[1.0], RHS=[2.0]" + ( + 'Variable "v1" data contents differ, at 1 points: ' + "@INDICES[(1,)] : LHS=[1.0], RHS=[2.0]" + ) ] check(errs, expected) @@ -424,13 +412,17 @@ def test_string_data(self, ndiffs): expected = [] if ndiffs == 1: expected = [ - 'Variable "vx" data contents differ, at 1 points: ' - "@INDICES[(1, 1)] : LHS=[b'h'], RHS=[b'X']" + ( + 'Variable "vx" data contents differ, at 1 points: ' + "@INDICES[(1, 1)] : LHS=[b'h'], RHS=[b'X']" + ) ] elif ndiffs == 2: expected = [ - 'Variable "vx" data contents differ, at 7 points: ' - "@INDICES[(1, 1), (3, 3), ...] : " - "LHS=[b'h', b'e', ...], RHS=[b'X', b'', ...]" + ( + 'Variable "vx" data contents differ, at 7 points: ' + "@INDICES[(1, 1), (3, 3), ...] : " + "LHS=[b'h', b'e', ...], RHS=[b'X', b'', ...]" + ) ] check(errs, expected) diff --git a/tests/unit/utils/dim_indexing/__init__.py b/tests/unit/utils/dim_indexing/__init__.py index 7ad084b..37b1cc6 100644 --- a/tests/unit/utils/dim_indexing/__init__.py +++ b/tests/unit/utils/dim_indexing/__init__.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Unit tests for :mod:`ncdata.utils._dim_indexing`.""" import numpy as np @@ -50,9 +54,7 @@ def make_dims_testdata(x, y): else: ydims = [] - data.variables.add( - NcVariable("Y", ydims, data=y, attributes={"role": "Y dim var"}) - ) + data.variables.add(NcVariable("Y", ydims, data=y, attributes={"role": "Y dim var"})) data.variables.add( NcVariable( @@ -68,9 +70,7 @@ def make_dims_testdata(x, y): else: xdims = [] - data.variables.add( - NcVariable("X", xdims, data=x, attributes={"role": "X dim var"}) - ) + data.variables.add(NcVariable("X", xdims, data=x, attributes={"role": "X dim var"})) xydims = ["Z"] + (["Y"] if y.ndim else []) + (["X"] if x.ndim else []) shape = [2] + ([len(y)] if y.ndim else []) + ([len(x)] if x.ndim else []) data.variables.add( @@ -97,7 +97,7 @@ def make_dims_testdata_1d(x): return data -class Slicekeys: # noqa: D101 +class Slicekeys: def __getitem__(self, keys): # noqa: D105 return keys diff --git a/tests/unit/utils/dim_indexing/test_Slicer.py b/tests/unit/utils/dim_indexing/test_Slicer.py index 4652cc2..f03c56c 100644 --- a/tests/unit/utils/dim_indexing/test_Slicer.py +++ b/tests/unit/utils/dim_indexing/test_Slicer.py @@ -1,4 +1,9 @@ -"""Tests for class :class:`ncdata.utils.Slicer`. +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +""" +Tests for class :class:`ncdata.utils.Slicer`. This is the "indirect" approach, with a Slicer object. """ diff --git a/tests/unit/utils/dim_indexing/test_index_by_dimensions.py b/tests/unit/utils/dim_indexing/test_index_by_dimensions.py index 099b1d7..a107814 100644 --- a/tests/unit/utils/dim_indexing/test_index_by_dimensions.py +++ b/tests/unit/utils/dim_indexing/test_index_by_dimensions.py @@ -1,4 +1,9 @@ -"""Tests for class :class:`ncdata.utils.index_by_dimension`. +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +""" +Tests for class :class:`ncdata.utils.index_by_dimension`. This is the more direct indexer approach. """ @@ -75,9 +80,7 @@ def test_integer_array(self): ) def test_boolean_array(self): - res1 = index_by_dimensions( - self.sample, X=[True, False, True, False, False] - ) + res1 = index_by_dimensions(self.sample, X=[True, False, True, False, False]) res2 = index_by_dimensions(self.sample, X=[0, 2]) assert dataset_differences(res1, res2) == [] @@ -91,9 +94,7 @@ def test_ellipsis_fail(self): with pytest.raises(ValueError, match=msg): index_by_dimensions(self.sample, Z=...) - @pytest.mark.parametrize( - "key", [np.newaxis, None], ids=["newaxis", "None"] - ) + @pytest.mark.parametrize("key", [np.newaxis, None], ids=["newaxis", "None"]) def test_newaxis_fail(self, key): msg = "Key for dimension 'Y' is np.newaxis / None.* not supported." with pytest.raises(ValueError, match=msg): diff --git a/tests/unit/utils/test_ncdata_copy.py b/tests/unit/utils/test_ncdata_copy.py index aff9734..4ef7208 100644 --- a/tests/unit/utils/test_ncdata_copy.py +++ b/tests/unit/utils/test_ncdata_copy.py @@ -1,4 +1,9 @@ -"""Tests for :func:`ncdata.utils.ncdata_copy`. +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +""" +Tests for :func:`ncdata.utils.ncdata_copy`. This is generic utility function version of the copy operation. """ @@ -13,22 +18,24 @@ def _ncdata_duplicate_object(d1: NcData, d2: NcData): """Find and return the first known duplicate objects between two NcData.""" dup = None - for var1, var2 in zip(d1.variables.values(), d2.variables.values()): + for var1, var2 in zip(d1.variables.values(), d2.variables.values(), strict=True): if var1 is var2: dup = var1 break if not dup: - for dim1, dim2 in zip(d1.dimensions.values(), d2.dimensions.values()): + for dim1, dim2 in zip( + d1.dimensions.values(), d2.dimensions.values(), strict=True + ): if dim1 is dim2: dup = dim1 break if not dup: - for attr1, attr2 in zip(d1.avals.values(), d2.avals.values()): + for attr1, attr2 in zip(d1.avals.values(), d2.avals.values(), strict=True): if attr1 is attr2: dup = attr1 break if not dup: - for grp1, grp2 in zip(d1.groups.values(), d2.groups.values()): + for grp1, grp2 in zip(d1.groups.values(), d2.groups.values(), strict=True): if grp1 is grp2: dup = grp1 break @@ -49,16 +56,12 @@ def test_empty(self): result = ncdata_copy(sample) assert not differences_or_duplicated_objects(sample, result) - @pytest.fixture() + @pytest.fixture def sample(self): - attrs = NameMap.from_items( - [NcAttribute("q", 3)], item_type=NcAttribute - ) + attrs = NameMap.from_items([NcAttribute("q", 3)], item_type=NcAttribute) dims = NameMap.from_items([NcDimension("x", 3)], item_type=NcDimension) data_array = np.array([1, 2, 3]) - var = NcVariable( - name="a", dimensions=("x"), data=data_array, attributes=attrs - ) + var = NcVariable(name="a", dimensions=("x"), data=data_array, attributes=attrs) sample = NcData( dimensions=dims, variables=[var], @@ -96,16 +99,10 @@ def test_sample_attribute_arraydata(self, sample): result = ncdata_copy(sample) rva = result.variables["a"] - assert ( - result.attributes["extra"].value - is not sample.attributes["extra"].value - ) and np.all( - result.attributes["extra"].value - == sample.attributes["extra"].value + assert result.attributes["extra"].value is not sample.attributes["extra"].value + assert np.all( + result.attributes["extra"].value == sample.attributes["extra"].value ) - assert ( - rva.attributes["xx2"].value is not sva.attributes["xx2"].value - ) and np.all( - rva.attributes["xx2"].value == sva.attributes["xx2"].value - ) + assert rva.attributes["xx2"].value is not sva.attributes["xx2"].value + assert np.all(rva.attributes["xx2"].value == sva.attributes["xx2"].value) diff --git a/tests/unit/utils/test_rename_dimension.py b/tests/unit/utils/test_rename_dimension.py index 8c92d3f..67df14d 100644 --- a/tests/unit/utils/test_rename_dimension.py +++ b/tests/unit/utils/test_rename_dimension.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Tests for :func:`ncdata.utils.rename_dimension`.""" import numpy as np @@ -7,10 +11,9 @@ from ncdata.utils import rename_dimension, save_errors -def make_saveable( - ncdata: NcData, _outer_dims: dict[str, NcDimension] | None = None -): - """Add missing dimensions + data, to make a sample NcData save-able. +def make_saveable(ncdata: NcData, _outer_dims: dict[str, NcDimension] | None = None): + """ + Add missing dimensions + data, to make a sample NcData save-able. Create any missing variable dimensions (length==2). Add any missing variable data arrays. @@ -23,7 +26,8 @@ def make_saveable( outer_dim_names = [dim.name for dim in _outer_dims.values()] def getdim(dimname): - """Fetch a known dimension by name. + """ + Fetch a known dimension by name. Check own, then "outer" ones, to correctly implement *scope masking*. """ @@ -84,7 +88,10 @@ def test_basic(self): def test_name_collision_fail(self): ncdata = self.ncdata - msg = "Cannot rename dimension 'x' to 'y', because a 'y' dimension already exists." + msg = ( + "Cannot rename dimension 'x' to 'y', " + "because a 'y' dimension already exists." + ) with pytest.raises(ValueError, match=msg): rename_dimension(ncdata, "x", "y") @@ -94,7 +101,9 @@ def test_name_collision_fail(self): def test_name_collision_ingroup_fail(self, innergroup): ncdata = self.ncdata grp = NcData(name="inner", dimensions=[NcDimension("z", 2)]) - msg = "Cannot rename dimension 'x' to 'z', because a 'z' dimension already exists" + msg = ( + "Cannot rename dimension 'x' to 'z', because a 'z' dimension already exists" + ) if innergroup: grp = NcData(name="main", groups=[grp]) msg += ' in group "/main/inner".' @@ -115,7 +124,7 @@ def test_name_ingroup_masked_nofail(self): # No error in this case, because the "z" is inside a group with its own "x". rename_dimension(ncdata, "x", "z") - @pytest.fixture() + @pytest.fixture def group_example(self, setup): ncdata = self.ncdata.copy() ncdata.groups.addall( @@ -134,7 +143,7 @@ def group_example(self, setup): ), ] ) - yield ncdata + return ncdata def test_groups(self, group_example): ncdata = group_example diff --git a/tests/unit/utils/test_save_errors.py b/tests/unit/utils/test_save_errors.py index fff1853..2ce875a 100644 --- a/tests/unit/utils/test_save_errors.py +++ b/tests/unit/utils/test_save_errors.py @@ -1,6 +1,8 @@ -""" -Tests for :mod:`ncdata.utils.save_errors` -""" +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Tests for :mod:`ncdata.utils.save_errors`.""" import re @@ -19,9 +21,7 @@ def _basic_testdata(): ncdata = NcData( name="test_ds", dimensions=[ - NcDimension( - "xxx", 2 - ), # unused first dim, so rename doesn't break variable + NcDimension("xxx", 2), # unused first dim, so rename doesn't break variable NcDimension("x", 3), ], variables=[ @@ -67,16 +67,14 @@ def component(self, request): @staticmethod def component_propname(component: str): """Return the property name corresponding to the component type.""" - if component == "attribute": + if component == "attribute": # noqa: SIM108 # Access the NameMap of NcAttributes (not the .attributes values map) propname = "attributes" else: propname = component + "s" return propname - @pytest.mark.parametrize( - "badnametype", ["None", "empty", "number", "object"] - ) + @pytest.mark.parametrize("badnametype", ["None", "empty", "number", "object"]) def test_bad_name_type(self, badnametype, component): bad_name = { "None": None, @@ -121,9 +119,7 @@ def test_key_name_mismatch(self, component): debug_errors(errors) assert len(errors) == 1 - msg = ( - f"{component} element {key!r} has a different element.name : 'qqq'" - ) + msg = f"{component} element {key!r} has a different element.name : 'qqq'" assert re.search(msg, errors[0]) @@ -250,7 +246,7 @@ def test_shape_mismatch(self): def test_missing_dims(self, extradims): ncdata = _basic_testdata() var = list(ncdata.variables.values())[0] - extradims = list(char for char in extradims) + extradims = list(extradims) var.dimensions = var.dimensions + tuple(extradims) errors = save_errors(ncdata) diff --git a/tests/unit/xarray/__init__.py b/tests/unit/xarray/__init__.py index c143809..afc3502 100644 --- a/tests/unit/xarray/__init__.py +++ b/tests/unit/xarray/__init__.py @@ -1 +1,5 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """Unit tests for :mod:`ncdata.xarray`.""" diff --git a/tests/unit/xarray/test_from_xarray.py b/tests/unit/xarray/test_from_xarray.py index dd502e1..78d05ed 100644 --- a/tests/unit/xarray/test_from_xarray.py +++ b/tests/unit/xarray/test_from_xarray.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Tests for :func:`ncdata.xarray.from_xarray`. @@ -36,9 +40,7 @@ def file_and_xarray_from_spec( def test_lazy_nocompute(tmp_path): - """ - Check that from_xarray converts lazy variable data without fetching. - """ + """Check that from_xarray converts lazy variable data without fetching.""" # Create a simple xarray dataset from a spec test_spec = { "dims": [dict(name="x", size=3)], @@ -75,9 +77,7 @@ def test_lazy_nocompute(tmp_path): def test_real_nocopy(tmp_path): - """ - Check that from_xarray converts real variable data directly without copying. - """ + """Check that from_xarray converts real variable data directly without copying.""" # Create a simple xarray dataset from a spec test_spec = { "dims": [dict(name="x", size=3)], @@ -108,9 +108,7 @@ def test_kwargs__unlimited_dims(tmp_path, unlim_dims): """Check xarray-save kwargs control : make the "x" dimension unlimited.""" test_spec = { "dims": [dict(name="x", size=3)], - "vars": [ - dict(name="var_x", dims=["x"], dtype=np.float32, data=[1.0, 3, 7]) - ], + "vars": [dict(name="var_x", dims=["x"], dtype=np.float32, data=[1.0, 3, 7])], } original_path = tmp_path / "input.nc" xrds = file_and_xarray_from_spec(original_path, test_spec) @@ -128,11 +126,7 @@ def test_kwargs__encoding__packing(tmp_path): """Repeat the above, checking use of the 'encoding' kwarg.""" test_spec = { "dims": [dict(name="x", size=3)], - "vars": [ - dict( - name="var_x", dims=["x"], dtype=np.float32, data=[1.0, 2, 3.3] - ) - ], + "vars": [dict(name="var_x", dims=["x"], dtype=np.float32, data=[1.0, 2, 3.3])], } original_path = tmp_path / "input.nc" xrds = file_and_xarray_from_spec(original_path, test_spec) diff --git a/tests/unit/xarray/test_to_xarray.py b/tests/unit/xarray/test_to_xarray.py index 592c975..78aec2d 100644 --- a/tests/unit/xarray/test_to_xarray.py +++ b/tests/unit/xarray/test_to_xarray.py @@ -1,3 +1,7 @@ +# Copyright SciTools contributors +# +# This file is part of SciTools/ncdata and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. """ Tests for :func:`ncdata.xarray.to_xarray`. @@ -26,9 +30,7 @@ def test_lazy_nocompute(): dask_array = da.from_array(monitored_array, chunks=(2,), meta=np.ndarray) ncdata = NcData( dimensions=[NcDimension("x", 3)], - variables=[ - NcVariable(name="var_x", dimensions=["x"], data=dask_array) - ], + variables=[NcVariable(name="var_x", dimensions=["x"], data=dask_array)], ) # convert to xarray.Dataset @@ -52,9 +54,7 @@ def test_real_nocopy(): real_numpy_data = np.array([1.23, 4, 7.3, 4.1], dtype=np.float32) ncdata = NcData( dimensions=[NcDimension("x", 3)], - variables=[ - NcVariable(name="var_x", dimensions=["x"], data=real_numpy_data) - ], + variables=[NcVariable(name="var_x", dimensions=["x"], data=real_numpy_data)], ) # convert to xarray.Dataset @@ -71,7 +71,6 @@ def test_real_nocopy(): @pytest.mark.parametrize("scaleandoffset", ["WITHscaling", "NOscaling"]) def test_kwargs__scaleandoffset(scaleandoffset): """Check the operation of kwargs (= xarray load controls).""" - # Make a test NcData dataset with a scaled-and-offset variable. raw_int_data = np.array([66, 75, 99], dtype=np.int16) ncdata = NcData(