diff --git a/CMakeLists.txt b/CMakeLists.txt index fcfde4ed..aae5363e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,6 @@ gz_find_package(AVUTIL REQUIRED_BY av PRETTY libavutil) # Find assimp gz_find_package(GzAssimp REQUIRED_BY graphics PRETTY assimp) - message(STATUS "-------------------------------------------\n") @@ -137,7 +136,7 @@ configure_file("${PROJECT_SOURCE_DIR}/cppcheck.suppress.in" ${PROJECT_BINARY_DIR}/cppcheck.suppress) gz_configure_build(QUIT_IF_BUILD_ERRORS - COMPONENTS av events geospatial graphics profiler testing) + COMPONENTS av events geospatial graphics io profiler testing) #============================================================================ # Create package information diff --git a/io/BUILD.bazel b/io/BUILD.bazel new file mode 100644 index 00000000..89a00dbe --- /dev/null +++ b/io/BUILD.bazel @@ -0,0 +1,76 @@ +load( + "//gz_bazel:build_defs.bzl", + "GZ_ROOT", + "GZ_VISIBILITY", + "generate_include_header", + "gz_export_header", +) + +package( + default_visibility = GZ_VISIBILITY, + features = [ + "-parse_headers", + "-layering_check", + ], +) + +public_headers_no_gen = glob([ + "include/gz/common/*.hh", +]) + +sources = glob( + ["src/*.cc"], + exclude = ["src/*_TEST.cc"], +) + +test_sources = glob(["src/*_TEST.cc"]) + +gz_export_header( + name = "include/gz/common/io/Export.hh", + export_base = "GZ_COMMON_IO", + lib_name = "gz-common-io", + visibility = ["//visibility:private"], +) + +generate_include_header( + name = "iohh_genrule", + out = "include/gz/common/io.hh", + hdrs = public_headers_no_gen + [ + "include/gz/common/io/Export.hh", + ], +) + +public_headers = public_headers_no_gen + [ + "include/gz/common/io/Export.hh", + "include/gz/common/io.hh", +] + +cc_library( + name = "io", + srcs = sources, + hdrs = public_headers, + includes = ["include"], + deps = [ + GZ_ROOT + "gz_common", + GZ_ROOT + "gz_math", + ], +) + +cc_binary( + name = "libgz-common5-io.so", + includes = ["include"], + linkopts = ["-Wl,-soname,libgz-common5-io.so"], + linkshared = True, + deps = [":events"], +) + +[cc_test( + name = src.replace("/", "_").replace(".cc", "").replace("src_", ""), + srcs = [src], + deps = [ + ":io", + GZ_ROOT + "gz_common/test:test_utils", + "@gtest", + "@gtest//:gtest_main", + ], +) for src in test_sources] diff --git a/io/include/CMakeLists.txt b/io/include/CMakeLists.txt new file mode 100644 index 00000000..a35a0475 --- /dev/null +++ b/io/include/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(gz) diff --git a/io/include/gz/CMakeLists.txt b/io/include/gz/CMakeLists.txt new file mode 100644 index 00000000..e4717b2d --- /dev/null +++ b/io/include/gz/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(common) diff --git a/io/include/gz/common/CMakeLists.txt b/io/include/gz/common/CMakeLists.txt new file mode 100644 index 00000000..82eb256f --- /dev/null +++ b/io/include/gz/common/CMakeLists.txt @@ -0,0 +1 @@ +gz_install_all_headers(COMPONENT io) diff --git a/io/include/gz/common/CSVStreams.hh b/io/include/gz/common/CSVStreams.hh new file mode 100644 index 00000000..58d3ecb4 --- /dev/null +++ b/io/include/gz/common/CSVStreams.hh @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#ifndef GZ_COMMON_CSVSTREAMS_HH_ +#define GZ_COMMON_CSVSTREAMS_HH_ + +#include +#include +#include + +#include + +#include + +namespace gz +{ + namespace common + { + /// \brief A CSV specification. + struct GZ_COMMON_IO_VISIBLE CSVDialect + { + /// Field delimiter character. + char delimiter; + + /// Row termination character. + char terminator; + + /// Field quoting character. + char quote; + + /// CSV dialect as expected by Unix tools. + static const CSVDialect Unix; + }; + + /// \brief Check CSV dialects for equality. + /// \param[in] _lhs Left-hand side CSV dialect. + /// \param[in] _rhs Right-hand side CSV dialect. + /// \return true if CSV dialects are equal, false otherwise. + bool GZ_COMMON_IO_VISIBLE operator==(const CSVDialect &_lhs, + const CSVDialect &_rhs); + + /// \brief A token in CSV data. + /// + /// Lexical specifications are typically dictated by a CSV dialect. + struct CSVToken + { + /// Token type. + enum { + TEXT = 0, ///< A pure text token (e.g. a letter). + QUOTE, ///< A field quoting token (e.g. a double quote). + DELIMITER, ///< A field delimiter token (e.g. a comma). + TERMINATOR ///< A row termination token (e.g. a newline). + } type; + + /// Token character. + char character; + }; + + /// \brief Extract a single token from an input stream of CSV data. + /// + /// If tokenization fails, the CSV data stream ``failbit`` will be set. + /// + /// \param[in] _stream A stream of CSV data to tokenize. + /// \param[out] _token Output CSV token to extract into. + /// \param[in] _dialect CSV data dialect. Defaults to the Unix dialect. + /// \return same CSV data stream. + GZ_COMMON_IO_VISIBLE std::istream &ExtractCSVToken( + std::istream &_stream, CSVToken &_token, + const CSVDialect &_dialect = CSVDialect::Unix); + + /// \brief Parse a single row from an input stream of CSV data. + /// + /// If parsing fails, the CSV data stream ``failbit`` will be set. + /// + /// \param[in] _stream CSV data stream to parse. + /// \param[out] _row Output CSV row to parse into. + /// \param[in] _dialect CSV data dialect. Defaults to the Unix dialect. + /// \returns same CSV data stream. + GZ_COMMON_IO_VISIBLE std::istream &ParseCSVRow( + std::istream &_stream, std::vector &_row, + const CSVDialect &_dialect = CSVDialect::Unix); + + /// \brief A single-pass row iterator on an input stream of CSV data. + /// + /// Similar to std::istream_iterator, this iterator parses a stream of + /// CSV data, one row at a time. \see `ParseCSVRow`. + class GZ_COMMON_IO_VISIBLE CSVIStreamIterator + { + public: using iterator_category = std::input_iterator_tag; + public: using value_type = std::vector; + public: using difference_type = std::ptrdiff_t; + public: using pointer = const value_type*; + public: using reference = const value_type&; + + /// \brief Construct an end-of-stream iterator. + public: CSVIStreamIterator(); + + /// \brief Construct an iterator over `_stream`. + /// + /// The first row will be read from the underlying stream to + /// initialize the iterator. If there are parsing errors while + /// reading, the underlying stream ``failbit`` will be set. + /// + /// \param[in] _stream A stream of CSV data to iterate. + /// \param[in] _dialect CSV data dialect. Defaults to the Unix dialect. + public: explicit CSVIStreamIterator( + std::istream &_stream, + const CSVDialect &_dialect = CSVDialect::Unix); + + /// \brief Read the next row from the underlying stream. + /// + /// If the read fails, the iterator becomes an end-of-stream iterator. + /// If there are parsing errors while reading, the underlying stream + /// ``failbit`` will be set. If the iterator already is an end-of-stream + /// iterator, behavior is undefined. + /// + /// \return A reference to the iterator once modified. + public: CSVIStreamIterator &operator++(); + + /// \brief Read the next row from the underlying stream. + /// + /// If the read fails, the iterator becomes an end-of-stream iterator. + /// If there are parsing errors while reading, the underlying stream + /// ``failbit`` will be set. If the iterator already is an end-of-stream + /// iterator, behavior is undefined. + /// + /// \return A copy of the iterator before modification. Note that, + /// while an iterator copy retains its state, the underlying stream + /// may still be advanced. + public: CSVIStreamIterator operator++(int); + + /// \brief Check for iterator equality. + /// \param[in] _other Iterator to compare with. + /// \return true if both iterators are end-of-stream iterators + /// or if both iterator wrap the same stream and use the same dialect, + /// false otherwise. + public: bool operator==(const CSVIStreamIterator &_other) const; + + /// \brief Check for iterator inequality. + /// \param[in] _other Iterator to compare with. + /// \return true if both iterators are not equal, false otherwise. + public: bool operator!=(const CSVIStreamIterator &_other) const; + + /// \brief Access current row. + /// + /// Behavior is undefined if the iterator is an end-of-stream iterator. + /// + /// \return reference to the current row. + public: reference operator*() const; + + /// \brief Access current row. + /// + /// Behavior is undefined if the iterator is an end-of-stream iterator. + /// + /// \return pointer to the current row. + public: pointer operator->() const; + + /// \brief Pointer to private data. + private: GZ_UTILS_IMPL_PTR(dataPtr) + }; + } +} + +#endif diff --git a/io/include/gz/common/DataFrame.hh b/io/include/gz/common/DataFrame.hh new file mode 100644 index 00000000..b5a124dc --- /dev/null +++ b/io/include/gz/common/DataFrame.hh @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#ifndef GZ_COMMON_DATAFRAME_HH_ +#define GZ_COMMON_DATAFRAME_HH_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace gz +{ + namespace common + { + /// \brief An abstract data frame. + /// + /// \tparam K Column key type + /// \tparam V Column value type + template + class DataFrame + { + /// \brief Check if column is present. + /// \param[in] _key Key to column to look up. + /// \return whether the given column is present + /// in the data frame. + public: bool Has(const K &_key) const + { + return this->storage.count(_key) > 0; + } + + /// \brief Fetch mutable reference to column. + /// \param[in] _key Key to column to look up. + /// \return Mutable reference to column in the + /// data frame. + public: V &operator[](const K &_key) + { + return this->storage[_key]; + } + + /// \brief Fetch immutable reference to column + /// \param[in] _key Key to column to look up. + /// \return Immutable reference to column in the + /// data frame. + public: const V &operator[](const K &_key) const + { + return this->storage.at(_key); + } + + /// \brief Data frame storage + private: std::unordered_map storage; + }; + + /// \brief Traits for IO of data frames comprised of time varying volumetric grids. + /// + /// \tparam K Data frame key type. + /// \tparam T Time coordinate type. + /// \tparam V Grid value type. + /// \tparam P Spatial dimensions type. + template + struct IO>> + { + /// \brief Read data frame from CSV data stream. + /// + /// \param[in] _begin Beginning-of-stream iterator to CSV data stream. + /// \param[in] _end End-of-stream iterator to CSV data stream. + /// \param[in] _timeColumnName CSV data column name to use as time + /// dimension. + /// \param[in] _spatialColumnNames CSV data columns' names to use + /// as spatial (x, y, z) dimensions, in that order. + /// \throws std::invalid_argument if the CSV data stream is empty, or + /// if the CSV data stream has no header, or if the given columns + /// cannot be found in the CSV data stream header. + /// \return data frame read. + static DataFrame> + ReadFrom(CSVIStreamIterator _begin, + CSVIStreamIterator _end, + const std::string &_timeColumnName, + const std::array &_spatialColumnNames) + { + if (_begin == _end) + { + throw std::invalid_argument("CSV data stream is empty"); + } + const std::vector &header = *_begin; + if (header.empty()) + { + throw std::invalid_argument("CSV data stream has no header"); + } + + auto it = std::find(header.begin(), header.end(), _timeColumnName); + if (it == header.end()) + { + std::stringstream sstream; + sstream << "CSV data stream has no '" + << _timeColumnName << "' column"; + throw std::invalid_argument(sstream.str()); + } + const size_t timeIndex = it - header.begin(); + + std::array spatialColumnIndices; + for (size_t i = 0; i < _spatialColumnNames.size(); ++i) + { + it = std::find(header.begin(), header.end(), _spatialColumnNames[i]); + if (it == header.end()) + { + std::stringstream sstream; + sstream << "CSV data stream has no '" + << _spatialColumnNames[i] << "' column"; + throw std::invalid_argument(sstream.str()); + } + spatialColumnIndices[i] = it - header.begin(); + } + + return ReadFrom(_begin, _end, timeIndex, spatialColumnIndices); + } + + /// \brief Read data frame from CSV data stream. + /// + /// \param[in] _begin Beginning-of-stream iterator to CSV data stream. + /// \param[in] _end End-of-stream iterator to CSV data stream. + /// \param[in] _timeColumnIndex CSV data column index to use as + /// time dimension. + /// \param[in] _spatialColumnIndices CSV data columns indices + /// to use as spatial (x, y, z) dimensions, in that order. + /// \throws std::invalid_argument if the CSV data stream is empty, or + /// if the CSV data stream has no header, or if the given columns + /// cannot be found in the CSV data stream header. + /// \return data frame read. + static DataFrame> + ReadFrom(CSVIStreamIterator _begin, + CSVIStreamIterator _end, + const size_t &_timeColumnIndex = 0, + const std::array &_spatialColumnIndices = {1, 2, 3}) + { + if (_begin == _end) + { + throw std::invalid_argument("CSV data stream is empty"); + } + std::vector dataColumnIndices(_begin->size()); + std::iota(dataColumnIndices.begin(), dataColumnIndices.end(), 0); + auto last = dataColumnIndices.end(); + for (size_t index : {_timeColumnIndex, _spatialColumnIndices[0], + _spatialColumnIndices[1], _spatialColumnIndices[2]}) + { + auto it = std::find(dataColumnIndices.begin(), last, index); + if (it == last) + { + std::stringstream sstream; + sstream << "Column index " << index << " is" + << "out of range for CSV data stream"; + throw std::invalid_argument(sstream.str()); + } + *it = *(--last); + } + dataColumnIndices.erase(last, dataColumnIndices.end()); + + using FactoryT = + math::InMemoryTimeVaryingVolumetricGridFactory; + std::vector factories(dataColumnIndices.size()); + for (auto it = _begin; it != _end; ++it) + { + const T time = IO::ReadFrom(it->at(_timeColumnIndex)); + const math::Vector3

position{ + IO

::ReadFrom(it->at(_spatialColumnIndices[0])), + IO

::ReadFrom(it->at(_spatialColumnIndices[1])), + IO

::ReadFrom(it->at(_spatialColumnIndices[2]))}; + + for (size_t i = 0; i < dataColumnIndices.size(); ++i) + { + const V value = IO::ReadFrom(it->at(dataColumnIndices[i])); + factories[i].AddPoint(time, position, value); + } + } + + DataFrame> df; + for (size_t i = 0; i < dataColumnIndices.size(); ++i) + { + const std::string key = !_begin->empty() ? + _begin->at(dataColumnIndices[i]) : + "var" + std::to_string(dataColumnIndices[i]); + df[IO::ReadFrom(key)] = factories[i].Build(); + } + return df; + } + }; + } +} +#endif diff --git a/io/include/gz/common/Io.hh b/io/include/gz/common/Io.hh new file mode 100644 index 00000000..434b10ba --- /dev/null +++ b/io/include/gz/common/Io.hh @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#ifndef GZ_COMMON_IO_HH_ +#define GZ_COMMON_IO_HH_ + +#include +#include + +namespace gz +{ + namespace common + { + /// \brief Traits for type-specific object I/O. + /// + /// To be fully specialized as needed. + template + struct IO + { + /// \brief Read object from stream. + /// + /// This default implementation relies on stream operator overloads. + /// + /// \param[in] _istream Stream to read object from. + /// \return object instance. + static T ReadFrom(std::istream &_istream) + { + T value; + _istream >> value; + return value; + } + + /// \brief Read object from string. + /// + /// This default implementation relies on stream operator overloads. + /// + /// \param[in] _string String to read object from. + /// \return object instance. + static T ReadFrom(const std::string &_string) + { + std::istringstream stream{_string}; + return ReadFrom(stream); + } + }; + + /// \brief Traits for string I/O. + template<> + struct IO + { + /// \brief Read object from stream. + /// + /// This default implementation relies on stream operator overloads. + /// + /// \param[in] _istream Stream to read object from. + /// \return object instance. + static std::string ReadFrom(std::istream &_istream) + { + std::string value; + _istream >> value; + return value; + } + + /// \brief Read string from string (copy as-is). + static std::string ReadFrom(std::string _string) + { + return _string; + } + }; + } +} + +#endif diff --git a/io/src/CMakeLists.txt b/io/src/CMakeLists.txt new file mode 100644 index 00000000..783bfd26 --- /dev/null +++ b/io/src/CMakeLists.txt @@ -0,0 +1,15 @@ +gz_get_libsources_and_unittests(sources gtest_sources) + +gz_add_component(io SOURCES ${sources} GET_TARGET_NAME io_target) + +target_link_libraries(${io_target} + PUBLIC + gz-math${GZ_MATH_VER}::gz-math${GZ_MATH_VER}) + +gz_build_tests( + TYPE UNIT + SOURCES ${gtest_sources} + LIB_DEPS + ${io_target} + gz-common${GZ_COMMON_VER}-testing +) diff --git a/io/src/CSVStreams.cc b/io/src/CSVStreams.cc new file mode 100644 index 00000000..a286e14c --- /dev/null +++ b/io/src/CSVStreams.cc @@ -0,0 +1,262 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +#include "gz/common/CSVStreams.hh" + +#include + +namespace gz +{ +namespace common +{ + +///////////////////////////////////////////////// +const CSVDialect CSVDialect::Unix = {',', '\n', '"'}; + +///////////////////////////////////////////////// +bool operator==(const CSVDialect &_lhs, const CSVDialect &_rhs) +{ + return (_lhs.delimiter == _rhs.delimiter && + _lhs.terminator == _rhs.terminator && + _lhs.quote == _rhs.quote); +} + +///////////////////////////////////////////////// +std::istream &ExtractCSVToken( + std::istream &_stream, CSVToken &_token, + const CSVDialect &_dialect) +{ + char character; + if (_stream.peek(), !_stream.fail() && _stream.eof()) + { + _token = {CSVToken::TERMINATOR, EOF}; + } + else if (_stream.get(character)) + { + if (character == _dialect.terminator) + { + _token = {CSVToken::TERMINATOR, character}; + } + else if (character == _dialect.delimiter) + { + _token = {CSVToken::DELIMITER, character}; + } + else if (character == _dialect.quote) + { + if (_stream.peek() == _dialect.quote) + { + _token = {CSVToken::TEXT, character}; + _stream.ignore(); + } + else + { + _token = {CSVToken::QUOTE, character}; + } + } + else + { + _token = {CSVToken::TEXT, character}; + } + } + return _stream; +} + +///////////////////////////////////////////////// +std::istream &ParseCSVRow( + std::istream &_stream, + std::vector &_row, + const CSVDialect &_dialect) +{ + std::stringstream text; + enum { + FIELD_START = 0, + ESCAPED_FIELD, + NONESCAPED_FIELD, + FIELD_END, + RECORD_END + } state = FIELD_START; + + _row.clear(); + + CSVToken token; + while (state != RECORD_END && ExtractCSVToken(_stream, token, _dialect)) + { + switch (state) + { + case FIELD_START: + if (token.type == CSVToken::QUOTE) + { + state = ESCAPED_FIELD; + break; + } + state = NONESCAPED_FIELD; + [[fallthrough]]; + case NONESCAPED_FIELD: + if (token.type == CSVToken::TEXT) + { + text << token.character; + break; + } + state = FIELD_END; + [[fallthrough]]; + case FIELD_END: + switch (token.type) + { + case CSVToken::DELIMITER: + _row.push_back(text.str()); + state = FIELD_START; + break; + case CSVToken::TERMINATOR: + if (token.character != EOF || !_row.empty() || text.tellp() > 0) + { + _row.push_back(text.str()); + state = RECORD_END; + break; + } + [[fallthrough]]; + default: + _stream.setstate(std::istream::failbit); + break; + } + text.str(""), text.clear(); + break; + case ESCAPED_FIELD: + if (token.type == CSVToken::QUOTE) + { + state = FIELD_END; + break; + } + if (token.type != CSVToken::TERMINATOR || token.character != EOF) + { + text << token.character; + break; + } + [[fallthrough]]; + default: + _stream.setstate(std::istream::failbit); + break; + } + } + return _stream; +} + +/// \brief Private data for the CSVIStreamIterator class +class CSVIStreamIterator::Implementation +{ + /// \brief Default constructor for end iterator. + public: Implementation() = default; + + /// \brief Constructor for begin iterator. + public: Implementation(std::istream &_stream, const CSVDialect &_dialect) + : stream(&_stream), dialect(_dialect) + { + } + + /// \brief Copy constructor. + public: Implementation(const Implementation &_other) + : stream(_other.stream), dialect(_other.dialect), row(_other.row) + { + } + + /// \brief Advance iterator to next row if possible. + public: void Next() + { + if (this->stream) + { + try + { + if (!ParseCSVRow(*this->stream, this->row, this->dialect)) + { + this->stream = nullptr; + } + } + catch (...) + { + this->stream = nullptr; + throw; + } + } + } + + /// \brief CSV data stream to iterate, if any. + public: std::istream *stream{nullptr}; + + /// \brief CSV dialect for data parsing. + public: CSVDialect dialect{}; + + /// \brief Current CSV data row. + public: std::vector row; +}; + +///////////////////////////////////////////////// +CSVIStreamIterator::CSVIStreamIterator() + : dataPtr(gz::utils::MakeImpl()) +{ +} + +///////////////////////////////////////////////// +CSVIStreamIterator::CSVIStreamIterator(std::istream &_stream, + const CSVDialect &_dialect) + : dataPtr(gz::utils::MakeImpl(_stream, _dialect)) +{ + this->dataPtr->Next(); +} + +///////////////////////////////////////////////// +bool CSVIStreamIterator::operator==(const CSVIStreamIterator &_other) const +{ + return this->dataPtr->stream == _other.dataPtr->stream && ( + this->dataPtr->stream == nullptr || + this->dataPtr->dialect == _other.dataPtr->dialect); +} + +///////////////////////////////////////////////// +bool CSVIStreamIterator::operator!=(const CSVIStreamIterator &_other) const +{ + return !(*this == _other); +} + +///////////////////////////////////////////////// +CSVIStreamIterator &CSVIStreamIterator::operator++() +{ + this->dataPtr->Next(); + return *this; +} + +///////////////////////////////////////////////// +// NOLINTNEXTLINE(readability/casting) +CSVIStreamIterator CSVIStreamIterator::operator++(int) +{ + CSVIStreamIterator it(*this); + this->dataPtr->Next(); + return it; +} + +///////////////////////////////////////////////// +const std::vector &CSVIStreamIterator::operator*() const +{ + return this->dataPtr->row; +} + +///////////////////////////////////////////////// +const std::vector *CSVIStreamIterator::operator->() const +{ + return &this->dataPtr->row; +} + +} +} diff --git a/io/src/CSVStreams_TEST.cc b/io/src/CSVStreams_TEST.cc new file mode 100644 index 00000000..ec653329 --- /dev/null +++ b/io/src/CSVStreams_TEST.cc @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#include +#include +#include +#include + +#include "gz/common/CSVStreams.hh" + +using namespace gz; +using namespace common; + +///////////////////////////////////////////////// +TEST(CSVStreams, CanExtractCSVTokens) +{ + std::stringstream sstream; + sstream << "\"a,\n\"\""; + + CSVToken token; + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::QUOTE); + EXPECT_EQ(token.character, '"'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TEXT); + EXPECT_EQ(token.character, 'a'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::DELIMITER); + EXPECT_EQ(token.character, ','); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TERMINATOR); + EXPECT_EQ(token.character, '\n'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TEXT); + EXPECT_EQ(token.character, '"'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TERMINATOR); + EXPECT_EQ(token.character, EOF); +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanParseCSVRows) +{ + { + std::stringstream sstream; + sstream << ","; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"", ""}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << "foo"; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"foo"}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << "foo" << std::endl; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"foo"}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << ",foo"; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"", "foo"}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << ",\"foo,bar\nbaz\","; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"", "foo,bar\nbaz", ""}; + EXPECT_EQ(row, expectedRow); + } +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanHandleInvalidCSVRows) +{ + { + std::stringstream sstream; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } + + { + std::stringstream sstream; + sstream << "\""; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } + + { + std::stringstream sstream; + sstream << "\"foo\"?"; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } + + { + std::stringstream sstream; + sstream << "foo\"bar\""; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanIterateValidCSV) +{ + { + std::stringstream sstream; + EXPECT_EQ(CSVIStreamIterator(sstream), + CSVIStreamIterator()); + } + + { + std::stringstream ss; + ss << std::endl; + const std::vector> expectedRows{{""}}; + const auto rows = std::vector>( + CSVIStreamIterator(ss), CSVIStreamIterator()); + EXPECT_EQ(expectedRows, rows); + EXPECT_TRUE(ss.eof()); + } + + { + std::stringstream ss; + ss << "foo,bar" << std::endl + << "bar," << std::endl + << ",foo" << std::endl + << "," << std::endl + << "baz,baz"; + const std::vector> expectedRows{ + {"foo", "bar"}, {"bar", ""}, {"", "foo"}, {"", ""}, {"baz", "baz"}}; + const auto rows = std::vector>( + CSVIStreamIterator(ss), CSVIStreamIterator()); + EXPECT_EQ(expectedRows, rows); + } +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanIterateInvalidCSVSafely) +{ + { + std::stringstream sstream; + sstream << "\"" << std::endl; + auto it = CSVIStreamIterator(sstream, CSVDialect::Unix); + EXPECT_EQ(it, CSVIStreamIterator()); + } + + { + std::stringstream sstream; + sstream.exceptions(std::stringstream::failbit); + sstream << "foo" << std::endl + << "\"bar" << std::endl; + auto it = CSVIStreamIterator(sstream, CSVDialect::Unix); + EXPECT_THROW({ ++it; }, std::stringstream::failure); + EXPECT_EQ(it, CSVIStreamIterator()); + } +} diff --git a/io/src/DataFrame_TEST.cc b/io/src/DataFrame_TEST.cc new file mode 100644 index 00000000..9100ae6d --- /dev/null +++ b/io/src/DataFrame_TEST.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +#include +#include +#include +#include + +#include + +#include "gz/common/CSVStreams.hh" +#include "gz/common/DataFrame.hh" +#include "gz/common/Filesystem.hh" + +using namespace gz; + +///////////////////////////////////////////////// +TEST(DataFrameTests, SimpleCSV) +{ + std::stringstream ss; + ss << "t,x,y,z,temperature" << std::endl + << "0,0,0,0,25.2" << std::endl + << "0,10,0,0,25.2" << std::endl + << "0,0,10,0,25.2" << std::endl + << "0,10,10,0,25.2" << std::endl + << "1,0,0,0,24.9" << std::endl + << "1,10,0,0,24.9" << std::endl + << "1,0,10,0,25.1" << std::endl + << "1,10,10,0,25.1" << std::endl; + + using DataT = + math::InMemoryTimeVaryingVolumetricGrid; + using DataFrameT = common::DataFrame; + const auto df = common::IO::ReadFrom( + common::CSVIStreamIterator(ss), + common::CSVIStreamIterator()); + + ASSERT_TRUE(df.Has("temperature")); + const DataT &temperatureData = df["temperature"]; + auto temperatureSession = temperatureData.StepTo( + temperatureData.CreateSession(), 0.5); + ASSERT_TRUE(temperatureSession.has_value()); + const math::Vector3d position{5., 5., 0.}; + auto temperature = temperatureData.LookUp( + temperatureSession.value(), position); + ASSERT_TRUE(temperature.has_value()); + EXPECT_DOUBLE_EQ(25.1, temperature.value()); +} + +///////////////////////////////////////////////// +TEST(DataFrameTests, ComplexCSV) +{ + std::stringstream ss; + ss << "timestamp,temperature,pressure,humidity,lat,lon,altitude" << std::endl + << "1658923062,13.1,101490,91,36.80029505,-121.788972517,0.8" << std::endl + << "1658923062,13,101485,88,36.80129505,-121.788972517,0.8" << std::endl + << "1658923062,13.1,101485,89,36.80029505,-121.789972517,0.8" << std::endl + << "1658923062,13.5,101490,92,36.80129505,-121.789972517,0.8" << std::endl; + + using DataT = + math::InMemoryTimeVaryingVolumetricGrid; + using DataFrameT = common::DataFrame; + const auto df = common::IO::ReadFrom( + common::CSVIStreamIterator(ss), common::CSVIStreamIterator(), + "timestamp", {"lat", "lon", "altitude"}); + EXPECT_TRUE(df.Has("temperature")); + EXPECT_TRUE(df.Has("humidity")); + ASSERT_TRUE(df.Has("pressure")); + + const DataT &pressureData = df["pressure"]; + auto pressureSession = pressureData.CreateSession(); + const math::Vector3d position{36.80079505, -121.789472517, 0.8}; + auto pressure = pressureData.LookUp(pressureSession, position); + ASSERT_TRUE(pressure.has_value()); + EXPECT_DOUBLE_EQ(101487.5, pressure.value()); +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d14341b9..bcb1ff4c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,6 +49,11 @@ gz_build_tests( # Used to make internal source file headers visible to the unit tests ${CMAKE_CURRENT_SOURCE_DIR}) +if(TARGET UNIT_DataFrame_TEST) + target_include_directories(UNIT_DataFrame_TEST PRIVATE + ${gz-math${GZ_MATH_VER}_INCLUDE_DIRS}) +endif() + if(TARGET UNIT_MovingWindowFilter_TEST) target_include_directories(UNIT_MovingWindowFilter_TEST PRIVATE ${gz-math${GZ_MATH_VER}_INCLUDE_DIRS})