From 2117b59ac3bf7e756b88d837850bd463f093ae82 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sun, 8 Sep 2024 10:59:18 -0600 Subject: [PATCH 1/8] Silence hypothesis complex warning Closes #2155 --- src/zarr/core/buffer/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 6aff47e8d3..5d77684f91 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +import warnings from abc import ABC, abstractmethod from typing import ( TYPE_CHECKING, @@ -457,7 +458,11 @@ def __repr__(self) -> str: return f"" def all_equal(self, other: Any) -> bool: - return bool((self._data == other).all()) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", r"invalid value encountered in equal", category=RuntimeWarning + ) + return bool((self._data == other).all()) def fill(self, value: Any) -> None: self._data.fill(value) From a70154eb373bebff0dc8b8ceb9d2a1439bfbca41 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sun, 8 Sep 2024 11:01:02 -0600 Subject: [PATCH 2/8] Use `array_equal` instead --- src/zarr/core/buffer/core.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 5d77684f91..c48a4a5191 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -import warnings from abc import ABC, abstractmethod from typing import ( TYPE_CHECKING, @@ -458,11 +457,7 @@ def __repr__(self) -> str: return f"" def all_equal(self, other: Any) -> bool: - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", r"invalid value encountered in equal", category=RuntimeWarning - ) - return bool((self._data == other).all()) + return np.array_equal(self._data, other, equal_nan=True) def fill(self, value: Any) -> None: self._data.fill(value) From eeaea6adbed4c686315b26c3a2d8e849eb8a688c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sun, 8 Sep 2024 17:13:59 -0600 Subject: [PATCH 3/8] Fix test --- src/zarr/core/buffer/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index c48a4a5191..295fb5f5a1 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -457,7 +457,10 @@ def __repr__(self) -> str: return f"" def all_equal(self, other: Any) -> bool: - return np.array_equal(self._data, other, equal_nan=True) + # use array_equal to obtain equal_nan=True functionality + data, other = np.broadcast_arrays(self._data, other) + result = np.array_equal(self._data, other, equal_nan=True) + return result def fill(self, value: Any) -> None: self._data.fill(value) From f21fcc2e4e1fac2534244372afc30915e3e84858 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 9 Sep 2024 10:53:12 -0600 Subject: [PATCH 4/8] Add nan fill value test --- tests/v3/test_array.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index fb726757dc..9f2069fbb1 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -4,6 +4,7 @@ import pytest from zarr import Array, Group +from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.store import LocalStore, MemoryStore @@ -135,3 +136,22 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str assert arr.fill_value == np.dtype(dtype_str).type(fill_value) assert arr.fill_value.dtype == arr.dtype + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: + shape = (10,) + arr = Array.create( + store=store, + shape=shape, + dtype=np.float64, + zarr_format=3, + chunk_shape=shape, + fill_value=np.nan, + ) + arr[:] = np.nan + + assert np.isnan(arr.fill_value) + assert arr.fill_value.dtype == arr.dtype + # all fill value chunk is an empty chunk, and should not be written + assert await store.get("c/0", prototype=default_buffer_prototype()) is None From b5e4f9cdbd1b04e4631e6ccecbe1ffd3fe5251c9 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 9 Sep 2024 10:57:47 -0600 Subject: [PATCH 5/8] better test --- tests/v3/test_array.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index 9f2069fbb1..2a16e142f9 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -4,7 +4,6 @@ import pytest from zarr import Array, Group -from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.store import LocalStore, MemoryStore @@ -154,4 +153,4 @@ async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: assert np.isnan(arr.fill_value) assert arr.fill_value.dtype == arr.dtype # all fill value chunk is an empty chunk, and should not be written - assert await store.get("c/0", prototype=default_buffer_prototype()) is None + assert not [a async for a in store.list_prefix("/")] From eee9a0899e11a1a90af1b9f7c119dafa97eccd34 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 16 Sep 2024 10:24:32 -0600 Subject: [PATCH 6/8] add equal_nan kwarg --- src/zarr/core/buffer/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 9bac4c7d5d..ba629befa1 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -462,10 +462,11 @@ def __len__(self) -> int: def __repr__(self) -> str: return f"" - def all_equal(self, other: Any) -> bool: + def all_equal(self, other: Any, equal_nan: bool = True) -> bool: + """Compare to `other` using np.array_equal.""" # use array_equal to obtain equal_nan=True functionality data, other = np.broadcast_arrays(self._data, other) - result = np.array_equal(self._data, other, equal_nan=True) + result = np.array_equal(self._data, other, equal_nan=equal_nan) return result def fill(self, value: Any) -> None: From 61448a850181d820ff92a2a497f9de58f78df7f7 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 16 Sep 2024 13:07:42 -0600 Subject: [PATCH 7/8] cleanup --- tests/v3/test_properties.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/v3/test_properties.py b/tests/v3/test_properties.py index 7a085c03b7..a78e9207bd 100644 --- a/tests/v3/test_properties.py +++ b/tests/v3/test_properties.py @@ -18,11 +18,6 @@ def test_roundtrip(data: st.DataObject) -> None: @given(data=st.data()) -# The filter warning here is to silence an occasional warning in NDBuffer.all_equal -# See https://github.com/zarr-developers/zarr-python/pull/2118#issuecomment-2310280899 -# Uncomment the next line to reproduce the original failure. -# @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/ndR2z7nkDZEDADWpBL4=') -@pytest.mark.filterwarnings("ignore::RuntimeWarning") def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(arrays()) nparray = zarray[:] @@ -37,11 +32,6 @@ def test_basic_indexing(data: st.DataObject) -> None: @given(data=st.data()) -# The filter warning here is to silence an occasional warning in NDBuffer.all_equal -# See https://github.com/zarr-developers/zarr-python/pull/2118#issuecomment-2310280899 -# Uncomment the next line to reproduce the original failure. -# @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/eLmF7qr/C5EDADZUBRM=') -@pytest.mark.filterwarnings("ignore::RuntimeWarning") def test_vindex(data: st.DataObject) -> None: zarray = data.draw(arrays()) nparray = zarray[:] From 65a8654b37906fc4610249f3df14d4fd7563fb8c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 17 Sep 2024 07:18:56 -0600 Subject: [PATCH 8/8] Update tests/v3/test_array.py Co-authored-by: David Stansby --- tests/v3/test_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index 12097267b7..b7beb63b1c 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -154,7 +154,7 @@ async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: assert np.isnan(arr.fill_value) assert arr.fill_value.dtype == arr.dtype # all fill value chunk is an empty chunk, and should not be written - assert not [a async for a in store.list_prefix("/")] + assert len([a async for a in store.list_prefix("/")]) == 0 @pytest.mark.parametrize("store", ("local",), indirect=["store"])