diff --git a/CMakeLists.txt b/CMakeLists.txt index 19a0949f9..39a345b9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ project (nix C CXX) set(VERSION_MAJOR 1) set(VERSION_MINOR 4) -set(VERSION_PATCH 3) +set(VERSION_PATCH 4) set(VERSION_ABI 1) diff --git a/nixio.spec b/nixio.spec index 7ace1330e..5392139cc 100644 --- a/nixio.spec +++ b/nixio.spec @@ -1,5 +1,5 @@ Name: nixio -Version: 1.4.3 +Version: 1.4.4 Release: 1%{?dist} Summary: IO-libray for nix data files @@ -56,6 +56,9 @@ cd build %changelog +* Tue Nov 26 2019 Jan Grewe - 1.4.0-1 +- version bump to 1.4.4 + * Thu Nov 01 2018 Jan Grewe - 1.4.0-1 - version bump to 1.4.3 diff --git a/src/util/dataAccess.cpp b/src/util/dataAccess.cpp index 7646f4b51..a07a740e1 100644 --- a/src/util/dataAccess.cpp +++ b/src/util/dataAccess.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -184,21 +185,96 @@ ndsize_t positionToIndex(double position, const string &unit, const RangeDimensi } +void getMaxExtent(const Dimension &dim, ndsize_t max_index, double &pos, double &ext) { + DimensionType dt = dim.dimensionType(); + if (dt == DimensionType::Sample) { + SampledDimension sd = dim.asSampledDimension(); + pos = sd.positionAt(0); + ext = sd.positionAt(max_index); + } else if (dt == DimensionType::Range) { + RangeDimension rd = dim.asRangeDimension(); + pos = rd.tickAt(0); + ext = rd.tickAt(max_index); + } else if (dt == DimensionType::Set) { + SetDimension sd = dim.asSetDimension(); + pos = 0.0; + if (max_index > pow(FLT_RADIX, std::numeric_limits::digits)) { + throw nix::OutOfBounds("dataAccess::fillPositionsExtents: shape cannot be cast to double without loss of precision. Please open an issue on github!"); + } + ext = static_cast(max_index); + } +} + + +vector> maximumExtents(const DataArray &array) { + vector dimensions = array.dimensions(); + vector> max_extents; + for (size_t i = 0; i < dimensions.size(); ++i) { + double pos, ext; + getMaxExtent(dimensions[i], array.dataExtent()[i]-1, pos, ext); + max_extents.emplace_back(pos, ext); + } + return max_extents; +} + + +string getDimensionUnit(const Dimension &dim) { + if (dim.dimensionType() == DimensionType::Set) { + return "none"; + } + if (dim.dimensionType() == DimensionType::Sample) { + SampledDimension sd = dim.asSampledDimension(); + string unit = sd.unit() ? *sd.unit() : "none"; + return unit; + } else if (dim.dimensionType() == DimensionType::Range) { + RangeDimension rd = dim.asRangeDimension(); + string unit = rd.unit() ? *rd.unit() : "none"; + return unit; + } + return "none"; +} + + void getOffsetAndCount(const Tag &tag, const DataArray &array, NDSize &offset, NDSize &count) { vector position = tag.position(); vector extent = tag.extent(); vector units = tag.units(); - NDSize temp_offset(position.size()); - NDSize temp_count(position.size(), 1); vector dimensions = array.dimensions(); - - if (array.dimensionCount() != position.size() || (extent.size() > 0 && extent.size() != array.dimensionCount())) { - throw runtime_error("Dimensionality of position or extent vector does not match dimensionality of data!"); + size_t dim_count = dimensions.size(); + NDSize shape = array.dataExtent(); + + if (extent.size() > 0 && (extent.size() != position.size())) { + throw IncompatibleDimensions("Size of position and extent do not match!", + "nix::util::getOffsetAndCount"); + } + vector> max_extents; + if (position.size() < dim_count) { + max_extents = maximumExtents(array); + } + if (extent.size() == 0) { + extent.resize(position.size(), 0.0); } - if (units.size() < position.size()) - units = vector(position.size(), "none"); - if (extent.size() < position.size()) - extent = vector(position.size(), 0.0); + while (position.size() > dim_count) { + position.pop_back(); + extent.pop_back(); + } + while (position.size() < dim_count) { + position.push_back(get<0>(max_extents[position.size()])); + extent.push_back(get<1>(max_extents[extent.size()])); + } + + if (units.size() == 0) { + units = std::vector(position.size(), "none"); + } + while (units.size() > position.size()) { + units.pop_back(); + } + while (units.size() < position.size()) { + units.push_back(getDimensionUnit(dimensions[units.size()])); + } + + NDSize temp_offset(position.size()); + NDSize temp_count(position.size(), 1); for (size_t i = 0; i < position.size(); ++i) { vector> indices = positionToIndex({position[i]}, {position[i] + extent[i]}, @@ -211,16 +287,22 @@ void getOffsetAndCount(const Tag &tag, const DataArray &array, NDSize &offset, N count = temp_count; } + void getOffsetAndCount(const MultiTag &tag, const DataArray &array, const vector &indices, vector &offsets, vector &counts) { DataArray positions = tag.positions(); DataArray extents = tag.extents(); NDSize position_size, extent_size; ndsize_t dimension_count = array.dimensionCount(); + vector dimensions = array.dimensions(); vector units = tag.units(); while (units.size() < dimension_count) { units.push_back("none"); + } + vector> max_extents; + if (position_size.size() < dimension_count) { + max_extents = maximumExtents(array); } if (positions) { position_size = positions.dataExtent(); @@ -228,19 +310,6 @@ void getOffsetAndCount(const MultiTag &tag, const DataArray &array, const vector if (extents) { extent_size = extents.dataExtent(); } - - if (position_size.size() == 1 && dimension_count != 1) { - throw IncompatibleDimensions("Number of dimensions in positions does not match dimensionality of data", - "util::getOffsetAndCount"); - } - if (position_size.size() > 1 && position_size[1] > dimension_count) { - throw IncompatibleDimensions("Number of dimensions in positions does not match dimensionality of data", - "util::getOffsetAndCount"); - } - if (extents && extent_size != position_size) { - throw IncompatibleDimensions("Number of dimensions in extents does not match dimensionality of data", - "util::getOffsetAndCount"); - } ndsize_t max_index = *max_element(indices.begin(), indices.end()); if (max_index >= positions.dataExtent()[0] || (extents && max_index >= extents.dataExtent()[0])) { throw OutOfBounds("Index out of bounds of positions or extents!", 0); @@ -252,20 +321,31 @@ void getOffsetAndCount(const MultiTag &tag, const DataArray &array, const vector NDSize temp_count(positions.dataExtent().size(), static_cast(1)); int dim_index = dimension_count > 1 ? 1 : 0; - temp_count[dim_index] = static_cast(dimension_count); + int count = dimension_count > 1 ? position_size[dim_index] : 1; + temp_count[dim_index] = static_cast(count); - vector dimensions = array.dimensions(); - vector> start_positions(dimensions.size()); - vector> end_positions(dimensions.size()); + vector> start_positions(dimension_count); + vector> end_positions(dimension_count); + vector offset, extent; for (size_t idx = 0; idx < indices.size(); ++idx) { temp_offset[0] = indices[idx]; - vector offset, extent; positions.getData(offset, temp_count, temp_offset); if (extents) { extents.getData(extent, temp_count, temp_offset); } else { extent.resize(offset.size(), 0.0); } + // add pos/extents if missing + while (offset.size() < dimensions.size()) { + offset.push_back(get<0>(max_extents[offset.size()])); + extent.push_back(get<1>(max_extents[extent.size()])); + } + // throw away info, if not needed + while (offset.size() > dimensions.size()) { + offset.pop_back(); + extent.pop_back(); + } + for (size_t dim_index = 0; dim_index < dimensions.size(); ++dim_index) { if (idx == 0) { start_positions[dim_index] = vector(indices.size()); @@ -428,10 +508,6 @@ DataView retrieveData(const Tag &tag, ndsize_t reference_index) { DataView retrieveData(const Tag &tag, const DataArray &array) { vector positions = tag.position(); vector extents = tag.extent(); - ndsize_t dimension_count = array.dimensionCount(); - if (positions.size() != dimension_count || (extents.size() > 0 && extents.size() != dimension_count)) { - throw IncompatibleDimensions("Number of dimensions in position or extent do not match dimensionality of data", "util::retrieveData"); - } NDSize offset, count; getOffsetAndCount(tag, array, offset, count); diff --git a/src/valid/checks.cpp b/src/valid/checks.cpp index c606788eb..d94f434e5 100644 --- a/src/valid/checks.cpp +++ b/src/valid/checks.cpp @@ -88,6 +88,7 @@ bool extentsMatchRefs::operator()(const DataArray &extents) const { bool extentsMatchRefs::operator()(const std::vector &extents) const { bool mismatch = false; + /* auto extSize = extents.size(); auto it = refs.begin(); while (!mismatch && (it != refs.end())) { @@ -95,7 +96,7 @@ bool extentsMatchRefs::operator()(const std::vector &extents) const { mismatch = extSize != arrayExtent.size(); ++it; } - + */ return !mismatch; } diff --git a/test/BaseTestDataAccess.cpp b/test/BaseTestDataAccess.cpp index a5432d35b..0be6ae3d8 100644 --- a/test/BaseTestDataAccess.cpp +++ b/test/BaseTestDataAccess.cpp @@ -521,3 +521,166 @@ void BaseTestDataAccess::testDataSlice() { b.deleteDataArray(twod_array); file.deleteBlock(b); } + + +void BaseTestDataAccess::testFlexibleTagging() { + nix::Block b = file.createBlock("flexible tagging", "nix.test"); + + // create dummy data + std::vector data(1000, 0.0); + for (size_t i = 0; i array_type_2d; + typedef array_type_2d::index index; + nix::NDSize data_shape_2d(2, 0); + data_shape_2d[0] = 100; + data_shape_2d[1] = 10; + array_type_2d data2d(boost::extents[data_shape_2d[0]][data_shape_2d[1]]); + for(index i = 0; i < 100; ++i) { + for(index j = 0; j < 10; ++j) { + data2d[i][j] = std::rand() % 100 + 1; + } + } + + typedef boost::multi_array array_type_3d; + typedef array_type_3d::index index3; + nix::NDSize data_shape_3d(3, 0); + data_shape_3d[0] = 100; + data_shape_3d[1] = 10; + data_shape_3d[2] = 5; + + array_type_3d data3d(boost::extents[100][10][5]); + for(index3 i = 0; i < 100; ++i) { + for(index3 j = 0; j < 10; ++j) { + for(index3 k = 0; k < 5; ++k) { + data3d[i][j][k] = std::rand() % 100 + 1; + } + } + } + // Create the DataArrays and store the data. + nix::DataArray array = b.createDataArray("1d random data", "nix.sampled", data); + nix::SampledDimension dim = array.appendSampledDimension(0.1); + dim.label("time"); + dim.unit("s"); + + nix::DataArray array2d = b.createDataArray("2d random data", "nix.sampled.2d", data2d); + dim = array2d.appendSampledDimension(1.); + dim.label("time"); + dim.unit("s"); + array2d.appendSetDimension(); + + nix::DataArray array3d = b.createDataArray("3d random data", "nix.sampled.3d", data3d); + dim = array3d.appendSampledDimension(1.); + dim.label("time"); + dim.unit("s"); + array3d.appendSetDimension(); + array3d.appendSetDimension(); + + // Tag, tagging 2 dimensions + nix::Tag tag = b.createTag("1stTag", "nix.segment", {25, 0}); + tag.extent({50, 5}); + tag.units({"s"}); + tag.addReference(array); + tag.addReference(array2d); + tag.addReference(array3d); + + nix::DataView view = tag.retrieveData("1d random data"); + nix::NDSize exp_shape({501}); + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = tag.retrieveData("2d random data"); + exp_shape = {51, 6}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = tag.retrieveData("3d random data"); + exp_shape = {51, 6, 5}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + // Tag, tagging 3 dims without extents, i.e. a point + nix::Tag ndTag = b.createTag("2ndTag", "nix.points", {25, 0, 0}); + ndTag.addReference(array); + ndTag.addReference(array2d); + ndTag.addReference(array3d); + + view = ndTag.retrieveData("1d random data"); + exp_shape = {1}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = ndTag.retrieveData("2d random data"); + exp_shape = {1, 1}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = ndTag.retrieveData("3d random data"); + exp_shape = {1, 1, 1}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + //Tag, tagging 3d dims but with explicit zero extents + nix::Tag rdTag = b.createTag("3rdTag11", "nix.points", {25, 0, 0}); + rdTag.extent({0.0, 0.0, 0.0}); + rdTag.addReference(array); + rdTag.addReference(array2d); + rdTag.addReference(array3d); + + view = rdTag.retrieveData("1d random data"); + exp_shape = {1}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = rdTag.retrieveData("2d random data"); + exp_shape = {1, 1}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = rdTag.retrieveData("3d random data"); + exp_shape = {1, 1, 1}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + nix::Tag failTag = b.createTag("failing tag", "nix.points", {25, 0, 0}); + failTag.extent({0.0, 0.0}); // this is invalid! + failTag.addReference(array); + failTag.addReference(array2d); + failTag.addReference(array3d); + + CPPUNIT_ASSERT_THROW(failTag.retrieveData("3d random data"), nix::IncompatibleDimensions); + + // MultiTag + typedef boost::multi_array pos_type; + typedef pos_type::index index4; + + pos_type pos_data(boost::extents[5][2]); + pos_type ext_data(boost::extents[5][2]); + for(index4 i = 0; i < 5; ++i) { + pos_data[i][0] = i * 15.0 + 5.00; + ext_data[i][0] = 10.0; + pos_data[i][1] = 1.0; + ext_data[i][1] = 2.0; + } + nix::DataArray positions = b.createDataArray("mtag positions", "nix.positions.2d", pos_data); + positions.appendSetDimension(); + positions.appendSetDimension(); + + nix::DataArray extents = b.createDataArray("mtag_extents", "nix.extents.2d", ext_data); + extents.appendSetDimension(); + extents.appendSetDimension(); + + nix::MultiTag mtag = b.createMultiTag("mtag", "segments", positions); + mtag.extents(extents); + + mtag.addReference(array); + mtag.addReference(array2d); + mtag.addReference(array3d); + + view = mtag.retrieveData(0, "1d random data"); + exp_shape = {101}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = mtag.retrieveData(0, "2d random data"); + exp_shape = {11, 3}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + view = mtag.retrieveData(0, "3d random data"); + exp_shape = {11, 3, 5}; + CPPUNIT_ASSERT(view.dataExtent() == exp_shape); + + file.deleteBlock(b); +} diff --git a/test/BaseTestDataAccess.hpp b/test/BaseTestDataAccess.hpp index d53b02760..277778459 100644 --- a/test/BaseTestDataAccess.hpp +++ b/test/BaseTestDataAccess.hpp @@ -36,6 +36,7 @@ class BaseTestDataAccess : public CPPUNIT_NS::TestFixture { void testMultiTagUnitSupport(); void testDataView(); void testDataSlice(); + void testFlexibleTagging(); }; #endif // NIX_BASETESTDATAACCESS_H diff --git a/test/TestValidate.cpp b/test/TestValidate.cpp index a3957e18b..240063740 100644 --- a/test/TestValidate.cpp +++ b/test/TestValidate.cpp @@ -313,10 +313,10 @@ void TestValidate::test() { must( array2, &nix::DataArray::dimensions, dimTicksMatchData(array2), "dimTicksMatchData(array)"), should(tag, &nix::Tag::position, extentsMatchPositions(extent), "extentsMatchPositions(extent)"), must( mtag, &nix::MultiTag::positions, extentsMatchPositions(extents), "extentsMatchPositions(extents)"), - must( tag, &nix::Tag::extent, extentsMatchRefs(refs), "extentsMatchRefs(refs); (tag)"), - should(mtag, &nix::MultiTag::extents, extentsMatchRefs(refs), "extentsMatchRefs(refs); (mtag)"), - must( tag, &nix::Tag::position, positionsMatchRefs(refs), "positionsMatchRefs(refs); (tag)"), - must(mtag, &nix::MultiTag::positions, positionsMatchRefs(refs), "positionsMatchRefs(refs); (mtag)"), + //must( tag, &nix::Tag::extent, extentsMatchRefs(refs), "extentsMatchRefs(refs); (tag)"), + //should(mtag, &nix::MultiTag::extents, extentsMatchRefs(refs), "extentsMatchRefs(refs); (mtag)"), + //must( tag, &nix::Tag::position, positionsMatchRefs(refs), "positionsMatchRefs(refs); (tag)"), + //must(mtag, &nix::MultiTag::positions, positionsMatchRefs(refs), "positionsMatchRefs(refs); (mtag)"), should(dim_range1, &nix::RangeDimension::unit, isAtomicUnit(), "isAtomicUnit(); (dim_range1)"), should(tag, &nix::Tag::units, isAtomicUnit(), "isAtomicUnit(); (tag)"), must(units_tmp, &tag_tmp::unit, isCompoundUnit(), "isCompoundUnit(); (units_tmp.unit)"), @@ -347,10 +347,10 @@ void TestValidate::test() { should(array1, &nix::DataArray::dimensions, dimLabelsMatchData(array1), "dimLabelsMatchData(array)"), must(tag, &nix::Tag::position, extentsMatchPositions(extent), "extentsMatchPositions(extent)"),// must( mtag, &nix::MultiTag::positions, extentsMatchPositions(extents), "extentsMatchPositions(extents)"), - must( tag, &nix::Tag::extent, extentsMatchRefs(refs), "extentsMatchRefs(refs); (tag)"), - must(mtag, &nix::MultiTag::extents, extentsMatchRefs(refs), "extentsMatchRefs(refs); (mtag)"), - must( tag, &nix::Tag::position, positionsMatchRefs(refs), "positionsMatchRefs(refs); (tag)"), - must(mtag, &nix::MultiTag::positions, positionsMatchRefs(refs), "positionsMatchRefs(refs); (mtag)"), + // must( tag, &nix::Tag::extent, extentsMatchRefs(refs), "extentsMatchRefs(refs); (tag)"), + //must(mtag, &nix::MultiTag::extents, extentsMatchRefs(refs), "extentsMatchRefs(refs); (mtag)"), + //must( tag, &nix::Tag::position, positionsMatchRefs(refs), "positionsMatchRefs(refs); (tag)"), + //must(mtag, &nix::MultiTag::positions, positionsMatchRefs(refs), "positionsMatchRefs(refs); (mtag)"), must(units_tmp, &tag_tmp::unit, isAtomicUnit(), "isAtomicUnit(); (units_tmp.unit)"), must(units_tmp, &tag_tmp::units, isAtomicUnit(), "isAtomicUnit(); (units_tmp.units)"), must(units_tmp, &tag_tmp::unit, isCompoundUnit(), "isCompoundUnit(); (units_tmp.unit)"), @@ -364,12 +364,12 @@ void TestValidate::test() { }); // std::cout << myResult; CPPUNIT_ASSERT(myResult.getWarnings().size() == 3); - CPPUNIT_ASSERT(myResult.getErrors().size() == 16); + CPPUNIT_ASSERT(myResult.getErrors().size() == 12); // this used to be 16 myResult = file.validate(); // std::cout << myResult; CPPUNIT_ASSERT_EQUAL(static_cast(0), myResult.getWarnings().size()); - CPPUNIT_ASSERT_EQUAL(static_cast(11), myResult.getErrors().size()); + CPPUNIT_ASSERT_EQUAL(static_cast(9), myResult.getErrors().size()); //this used to be 11 // uncomment this to have debug info // std::cout << myResult; diff --git a/test/hdf5/TestDataAccessHDF5.hpp b/test/hdf5/TestDataAccessHDF5.hpp index 667c78014..1c56b2792 100644 --- a/test/hdf5/TestDataAccessHDF5.hpp +++ b/test/hdf5/TestDataAccessHDF5.hpp @@ -27,6 +27,7 @@ class TestDataAccessHDF5 : public BaseTestDataAccess { CPPUNIT_TEST(testMultiTagUnitSupport); CPPUNIT_TEST(testDataView); CPPUNIT_TEST(testDataSlice); + CPPUNIT_TEST(testFlexibleTagging); CPPUNIT_TEST_SUITE_END (); public: