From 3012471baced37b44ad05a00243891368ae2b0eb Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Fri, 25 Aug 2023 11:14:36 -0500 Subject: [PATCH 1/4] Use a smaller temporary buffer for GAMER --- yt/frontends/gamer/io.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/yt/frontends/gamer/io.py b/yt/frontends/gamer/io.py index 9287ed1dd1..099bff9f9d 100644 --- a/yt/frontends/gamer/io.py +++ b/yt/frontends/gamer/io.py @@ -121,24 +121,20 @@ def _read_fluid_selection(self, chunks, selector, fields, size): start = (gs[0].id) * self.pgroup end = (gs[-1].id + 1) * self.pgroup buf = ds[start:end, :, :, :] - ngrid = len(gs) - data = np.empty((ngrid, ps2, ps2, ps2), dtype=self._field_dtype) - - for g in range(ngrid): - pid0 = g * self.pgroup - data[g, 0:ps1, 0:ps1, 0:ps1] = buf[pid0 + 0, :, :, :] - data[g, 0:ps1, 0:ps1, ps1:ps2] = buf[pid0 + 1, :, :, :] - data[g, 0:ps1, ps1:ps2, 0:ps1] = buf[pid0 + 2, :, :, :] - data[g, ps1:ps2, 0:ps1, 0:ps1] = buf[pid0 + 3, :, :, :] - data[g, 0:ps1, ps1:ps2, ps1:ps2] = buf[pid0 + 4, :, :, :] - data[g, ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 5, :, :, :] - data[g, ps1:ps2, 0:ps1, ps1:ps2] = buf[pid0 + 6, :, :, :] - data[g, ps1:ps2, ps1:ps2, ps1:ps2] = buf[pid0 + 7, :, :, :] - - data = data.transpose() + data = np.empty((ps2, ps2, ps2), dtype=self._field_dtype) for i, g in enumerate(gs): - offset += g.select(selector, data[..., i], rv[field], offset) + pid0 = i * self.pgroup + data[0:ps1, 0:ps1, 0:ps1] = buf[pid0 + 0, :, :, :] + data[0:ps1, 0:ps1, ps1:ps2] = buf[pid0 + 1, :, :, :] + data[0:ps1, ps1:ps2, 0:ps1] = buf[pid0 + 2, :, :, :] + data[ps1:ps2, 0:ps1, 0:ps1] = buf[pid0 + 3, :, :, :] + data[0:ps1, ps1:ps2, ps1:ps2] = buf[pid0 + 4, :, :, :] + data[ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 5, :, :, :] + data[ps1:ps2, 0:ps1, ps1:ps2] = buf[pid0 + 6, :, :, :] + data[ps1:ps2, ps1:ps2, ps1:ps2] = buf[pid0 + 7, :, :, :] + offset += g.select(selector, data, rv[field], offset) + return rv def _read_chunk_data(self, chunk, fields): From 68f57879869d3606b1a0dc38d959d5504199e861 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Fri, 25 Aug 2023 12:02:14 -0500 Subject: [PATCH 2/4] Update yt/frontends/gamer/io.py --- yt/frontends/gamer/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/gamer/io.py b/yt/frontends/gamer/io.py index 099bff9f9d..c44c8634b7 100644 --- a/yt/frontends/gamer/io.py +++ b/yt/frontends/gamer/io.py @@ -133,7 +133,7 @@ def _read_fluid_selection(self, chunks, selector, fields, size): data[ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 5, :, :, :] data[ps1:ps2, 0:ps1, ps1:ps2] = buf[pid0 + 6, :, :, :] data[ps1:ps2, ps1:ps2, ps1:ps2] = buf[pid0 + 7, :, :, :] - offset += g.select(selector, data, rv[field], offset) + offset += g.select(selector, data.T, rv[field], offset) return rv From 79f3471f5e70fcdb85f1806085323e67864c72ed Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Fri, 25 Aug 2023 15:32:00 -0500 Subject: [PATCH 3/4] Use a cache and specialize with grid selector --- yt/frontends/gamer/io.py | 44 ++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/yt/frontends/gamer/io.py b/yt/frontends/gamer/io.py index c44c8634b7..4206f2ad04 100644 --- a/yt/frontends/gamer/io.py +++ b/yt/frontends/gamer/io.py @@ -1,8 +1,9 @@ +from functools import lru_cache from itertools import groupby import numpy as np -from yt.geometry.selection_routines import AlwaysSelector +from yt.geometry.selection_routines import AlwaysSelector, GridSelector from yt.utilities.io_handler import BaseIOHandler from yt.utilities.logger import ytLogger as mylog @@ -91,13 +92,39 @@ def _read_particle_fields(self, chunks, ptf, selector): data = self._group_particle[field][start:end] yield (ptype, field), data[mask] + @lru_cache(maxsize=8) # noqa + def _get_grid_field(self, field_name): + return self._group_grid[field_name] + def _read_fluid_selection(self, chunks, selector, fields, size): chunks = list(chunks) # generator --> list if any((ftype != "gamer" for ftype, fname in fields)): raise NotImplementedError + # shortcuts + ps2 = self.patch_size + ps1 = ps2 // 2 + data = np.empty((ps2, ps2, ps2), dtype=self._field_dtype) rv = {} + + if isinstance(selector, GridSelector): + if not (len(chunks) == len(chunks[0].objs) == 1): + raise RuntimeError + g = chunks[0].objs[0] + for ftype, fname in fields: + buf = self._get_grid_field(fname)[g.id : g.id + 8, ...] + data[0:ps1, 0:ps1, 0:ps1] = buf[0, :, :, :] + data[ps1:ps2, 0:ps1, 0:ps1] = buf[1, :, :, :] + data[0:ps1, ps1:ps2, 0:ps1] = buf[2, :, :, :] + data[0:ps1, 0:ps1, ps1:ps2] = buf[3, :, :, :] + data[ps1:ps2, ps1:ps2, 0:ps1] = buf[4, :, :, :] + data[0:ps1, ps1:ps2, ps1:ps2] = buf[5, :, :, :] + data[ps1:ps2, 0:ps1, ps1:ps2] = buf[6, :, :, :] + data[ps1:ps2, ps1:ps2, ps1:ps2] = buf[7, :, :, :] + rv[(ftype, fname)] = data.copy() + return rv + for field in fields: rv[field] = np.empty(size, dtype=self._field_dtype) @@ -109,10 +136,6 @@ def _read_fluid_selection(self, chunks, selector, fields, size): ng, ) - # shortcuts - ps2 = self.patch_size - ps1 = ps2 // 2 - for field in fields: ds = self._group_grid[field[1]] offset = 0 @@ -121,19 +144,18 @@ def _read_fluid_selection(self, chunks, selector, fields, size): start = (gs[0].id) * self.pgroup end = (gs[-1].id + 1) * self.pgroup buf = ds[start:end, :, :, :] - data = np.empty((ps2, ps2, ps2), dtype=self._field_dtype) for i, g in enumerate(gs): pid0 = i * self.pgroup data[0:ps1, 0:ps1, 0:ps1] = buf[pid0 + 0, :, :, :] - data[0:ps1, 0:ps1, ps1:ps2] = buf[pid0 + 1, :, :, :] + data[ps1:ps2, 0:ps1, 0:ps1] = buf[pid0 + 1, :, :, :] data[0:ps1, ps1:ps2, 0:ps1] = buf[pid0 + 2, :, :, :] - data[ps1:ps2, 0:ps1, 0:ps1] = buf[pid0 + 3, :, :, :] - data[0:ps1, ps1:ps2, ps1:ps2] = buf[pid0 + 4, :, :, :] - data[ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 5, :, :, :] + data[0:ps1, 0:ps1, ps1:ps2] = buf[pid0 + 3, :, :, :] + data[ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 4, :, :, :] + data[0:ps1, ps1:ps2, ps1:ps2] = buf[pid0 + 5, :, :, :] data[ps1:ps2, 0:ps1, ps1:ps2] = buf[pid0 + 6, :, :, :] data[ps1:ps2, ps1:ps2, ps1:ps2] = buf[pid0 + 7, :, :, :] - offset += g.select(selector, data.T, rv[field], offset) + offset += g.select(selector, data, rv[field], offset) return rv From d1a1d7366ac573faecf1d63298ea15ae6d35976d Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Fri, 25 Aug 2023 17:27:37 -0500 Subject: [PATCH 4/4] Undo my nonsensical transpose attempt --- yt/frontends/gamer/io.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/yt/frontends/gamer/io.py b/yt/frontends/gamer/io.py index 4206f2ad04..71547833f1 100644 --- a/yt/frontends/gamer/io.py +++ b/yt/frontends/gamer/io.py @@ -115,14 +115,14 @@ def _read_fluid_selection(self, chunks, selector, fields, size): for ftype, fname in fields: buf = self._get_grid_field(fname)[g.id : g.id + 8, ...] data[0:ps1, 0:ps1, 0:ps1] = buf[0, :, :, :] - data[ps1:ps2, 0:ps1, 0:ps1] = buf[1, :, :, :] + data[0:ps1, 0:ps1, ps1:ps2] = buf[1, :, :, :] data[0:ps1, ps1:ps2, 0:ps1] = buf[2, :, :, :] - data[0:ps1, 0:ps1, ps1:ps2] = buf[3, :, :, :] - data[ps1:ps2, ps1:ps2, 0:ps1] = buf[4, :, :, :] - data[0:ps1, ps1:ps2, ps1:ps2] = buf[5, :, :, :] + data[ps1:ps2, 0:ps1, 0:ps1] = buf[3, :, :, :] + data[0:ps1, ps1:ps2, ps1:ps2] = buf[4, :, :, :] + data[ps1:ps2, ps1:ps2, 0:ps1] = buf[5, :, :, :] data[ps1:ps2, 0:ps1, ps1:ps2] = buf[6, :, :, :] data[ps1:ps2, ps1:ps2, ps1:ps2] = buf[7, :, :, :] - rv[(ftype, fname)] = data.copy() + rv[(ftype, fname)] = data.transpose() return rv for field in fields: @@ -148,14 +148,14 @@ def _read_fluid_selection(self, chunks, selector, fields, size): for i, g in enumerate(gs): pid0 = i * self.pgroup data[0:ps1, 0:ps1, 0:ps1] = buf[pid0 + 0, :, :, :] - data[ps1:ps2, 0:ps1, 0:ps1] = buf[pid0 + 1, :, :, :] + data[0:ps1, 0:ps1, ps1:ps2] = buf[pid0 + 1, :, :, :] data[0:ps1, ps1:ps2, 0:ps1] = buf[pid0 + 2, :, :, :] - data[0:ps1, 0:ps1, ps1:ps2] = buf[pid0 + 3, :, :, :] - data[ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 4, :, :, :] - data[0:ps1, ps1:ps2, ps1:ps2] = buf[pid0 + 5, :, :, :] + data[ps1:ps2, 0:ps1, 0:ps1] = buf[pid0 + 3, :, :, :] + data[0:ps1, ps1:ps2, ps1:ps2] = buf[pid0 + 4, :, :, :] + data[ps1:ps2, ps1:ps2, 0:ps1] = buf[pid0 + 5, :, :, :] data[ps1:ps2, 0:ps1, ps1:ps2] = buf[pid0 + 6, :, :, :] data[ps1:ps2, ps1:ps2, ps1:ps2] = buf[pid0 + 7, :, :, :] - offset += g.select(selector, data, rv[field], offset) + offset += g.select(selector, data.T, rv[field], offset) return rv