(API) Generate bagfile metadata in Writer

rosbag2/CMakeLists.txt

@@ -15,6 +15,11 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
  add_compile_options(-Wall -Wextra -Wpedantic -Werror)
 endif()
 
+# Windows supplies macros for min and max by default. We should only use min and max from stl
+if(WIN32)
+ add_definitions(-DNOMINMAX)
+endif()
+
 option(DISABLE_SANITIZERS "disables the use of gcc saniztizers")
 # Only enable sanitizers on x64 architectures
 # https:/google/sanitizers/issues/794

rosbag2/include/rosbag2/writer.hpp

@@ -17,6 +17,8 @@
 
 #include <memory>
 #include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "rosbag2_storage/metadata_io.hpp"
 #include "rosbag2_storage/storage_factory.hpp"
@@ -106,8 +108,19 @@ class ROSBAG2_PUBLIC Writer
  // Used in bagfile splitting; specifies the best-effort maximum sub-section of a bagfile in bytes.
  uint64_t max_bagfile_size_;
 
+ // Used to track topic -> message count
+ std::unordered_map<std::string, TopicInformation> topics_names_to_info_;
+
+ rosbag2_storage::BagMetadata metadata_;
+
  // Checks if the current recording bagfile needs to be split and rolled over to a new file.
  bool should_split_bagfile() const;
+
+ // Prepares the metadata by setting initial values.
+ void init_metadata();
+
+ // Record TopicInformation into metadata
+ void finalize_metadata();
 };
 
 } // namespace rosbag2

rosbag2/src/rosbag2/writer.cpp

@@ -14,6 +14,10 @@
 
 #include "rosbag2/writer.hpp"
 
+#include <rosbag2_storage/filesystem_helper.hpp>
+
+#include <algorithm>
+#include <chrono>
 #include <memory>
 #include <stdexcept>
 #include <string>
@@ -34,19 +38,31 @@ Writer::Writer(
  storage_(nullptr),
  metadata_io_(std::move(metadata_io)),
  converter_(nullptr),
- max_bagfile_size_(rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT)
+ max_bagfile_size_(rosbag2_storage::storage_interfaces::MAX_BAGFILE_SIZE_NO_SPLIT),
+ topics_names_to_info_(),
+ metadata_()
 {}
 
 Writer::~Writer()
 {
  if (!uri_.empty()) {
- metadata_io_->write_metadata(uri_, storage_->get_metadata());
+ finalize_metadata();
+ metadata_io_->write_metadata(uri_, metadata_);
  }
 
  storage_.reset(); // Necessary to ensure that the storage is destroyed before the factory
  storage_factory_.reset();
 }
 
+void Writer::init_metadata()
+{
+ metadata_ = rosbag2_storage::BagMetadata{};
+ metadata_.storage_identifier = storage_->get_storage_identifier();
+ metadata_.starting_time = std::chrono::time_point<std::chrono::high_resolution_clock>(
+ std::chrono::nanoseconds::max());
+ metadata_.relative_file_paths = {storage_->get_relative_path()};
+}
+
 void Writer::open(
  const StorageOptions & storage_options,
  const ConverterOptions & converter_options)
@@ -63,7 +79,10 @@ void Writer::open(
  if (!storage_) {
  throw std::runtime_error("No storage could be initialized. Abort");
  }
+
  uri_ = storage_options.uri;
+
+ init_metadata();
 }
 
 void Writer::create_topic(const TopicMetadata & topic_with_type)
@@ -76,7 +95,24 @@ void Writer::create_topic(const TopicMetadata & topic_with_type)
  converter_->add_topic(topic_with_type.name, topic_with_type.type);
  }
 
- storage_->create_topic(topic_with_type);
+ if (topics_names_to_info_.find(topic_with_type.name) ==
+ topics_names_to_info_.end())
+ {
+ rosbag2_storage::TopicInformation info{};
+ info.topic_metadata = topic_with_type;
+
+ const auto insert_res = topics_names_to_info_.insert(
+ std::make_pair(topic_with_type.name, info));
+
+ if (!insert_res.second) {
+ std::stringstream errmsg;
+ errmsg << "Failed to insert topic \"" << topic_with_type.name << "\"!";
+
+ throw std::runtime_error(errmsg.str());
+ }
+
+ storage_->create_topic(topic_with_type);
+ }
 }
 
 void Writer::remove_topic(const TopicMetadata & topic_with_type)
@@ -85,7 +121,15 @@ void Writer::remove_topic(const TopicMetadata & topic_with_type)
  throw std::runtime_error("Bag is not open. Call open() before removing.");
  }
 
- storage_->remove_topic(topic_with_type);
+ if (topics_names_to_info_.erase(topic_with_type.name) > 0) {
+ storage_->remove_topic(topic_with_type);
+ } else {
+ std::stringstream errmsg;
+ errmsg << "Failed to remove the non-existing topic \"" <<
+ topic_with_type.name << "\"!";
+
+ throw std::runtime_error(errmsg.str());
+ }
 }
 
 void Writer::write(std::shared_ptr<SerializedBagMessage> message)
@@ -94,6 +138,16 @@ void Writer::write(std::shared_ptr<SerializedBagMessage> message)
  throw std::runtime_error("Bag is not open. Call open() before writing.");
  }
 
+ // Update the message count for the Topic.
+ ++topics_names_to_info_.at(message->topic_name).message_count;
+
+ const auto message_timestamp = std::chrono::time_point<std::chrono::high_resolution_clock>(
+ std::chrono::nanoseconds(message->time_stamp));
+ metadata_.starting_time = std::min(metadata_.starting_time, message_timestamp);
+
+ const auto duration = message_timestamp - metadata_.starting_time;
+ metadata_.duration = std::max(metadata_.duration, duration);
+
  storage_->write(converter_ ? converter_->convert(message) : message);
 }
 
@@ -105,4 +159,23 @@ bool Writer::should_split_bagfile() const
  return storage_->get_bagfile_size() > max_bagfile_size_;
  }
 }
+
+void Writer::finalize_metadata()
+{
+ metadata_.bag_size = 0;
+
+ for (const auto & path : metadata_.relative_file_paths) {
+ metadata_.bag_size += rosbag2_storage::FilesystemHelper::get_file_size(path);
+ }
+
+ metadata_.topics_with_message_count.clear();
+ metadata_.topics_with_message_count.reserve(topics_names_to_info_.size());
+ metadata_.message_count = 0;
+
+ for (const auto & topic : topics_names_to_info_) {
+ metadata_.topics_with_message_count.push_back(topic.second);
+ metadata_.message_count += topic.second.message_count;
+ }
+}
+
 } // namespace rosbag2

rosbag2/test/rosbag2/mock_storage.hpp

@@ -38,6 +38,7 @@ class MockStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterfa
  MOCK_METHOD0(get_all_topics_and_types, std::vector<rosbag2_storage::TopicMetadata>());
  MOCK_METHOD0(get_metadata, rosbag2_storage::BagMetadata());
  MOCK_CONST_METHOD0(get_bagfile_size, uint64_t());
+ MOCK_CONST_METHOD0(get_relative_path, std::string());
  MOCK_CONST_METHOD0(get_storage_identifier, std::string());
 };
 

rosbag2_converter_default_plugins/CMakeLists.txt

@@ -15,6 +15,11 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
  add_compile_options(-Wall -Wextra -Wpedantic)
 endif()
 
+# Windows supplies macros for min and max by default. We should only use min and max from stl
+if(WIN32)
+ add_definitions(-DNOMINMAX)
+endif()
+
 find_package(ament_cmake REQUIRED)
 find_package(ament_index_cpp REQUIRED)
 find_package(poco_vendor REQUIRED)

rosbag2_storage/CMakeLists.txt

@@ -15,6 +15,11 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
  add_compile_options(-Wall -Wextra -Wpedantic -Werror)
 endif()
 
+# Windows supplies macros for min and max by default. We should only use min and max from stl
+if(WIN32)
+ add_definitions(-DNOMINMAX)
+endif()
+
 find_package(ament_cmake REQUIRED)
 find_package(pluginlib REQUIRED)
 find_package(rcutils REQUIRED)

rosbag2_storage/include/rosbag2_storage/storage_interfaces/base_info_interface.hpp

@@ -16,6 +16,8 @@
 #ifndef ROSBAG2_STORAGE__STORAGE_INTERFACES__BASE_INFO_INTERFACE_HPP_
 #define ROSBAG2_STORAGE__STORAGE_INTERFACES__BASE_INFO_INTERFACE_HPP_
 
+#include <string>
+
 #include "rosbag2_storage/bag_metadata.hpp"
 #include "rosbag2_storage/visibility_control.hpp"
 
@@ -30,6 +32,13 @@ class ROSBAG2_STORAGE_PUBLIC BaseInfoInterface
  virtual ~BaseInfoInterface() = default;
 
  virtual BagMetadata get_metadata() = 0;
+
+ /**
+ * Retrieves the relative path to the backing of the storage plugin.
+ *
+ * \returns the relative path.
+ */
+ virtual std::string get_relative_path() const = 0;
 };
 
 } // namespace storage_interfaces

rosbag2_storage/test/rosbag2_storage/test_constants.hpp

@@ -0,0 +1,27 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#ifndef ROSBAG2_STORAGE__TEST_CONSTANTS_HPP_
+#define ROSBAG2_STORAGE__TEST_CONSTANTS_HPP_
+
+namespace test_constants
+{
+constexpr const char * const READ_WRITE_PLUGIN_IDENTIFIER = "ReadWritePlugin";
+constexpr const char * const READ_ONLY_PLUGIN_IDENTIFIER = "ReadOnlyPlugin";
+constexpr const char * const DUMMY_FILEPATH = "/path/to/storage";
+constexpr const uint64_t MAX_BAGFILE_SIZE = 0;
+}
+
+#endif // ROSBAG2_STORAGE__TEST_CONSTANTS_HPP_

rosbag2_storage/test/rosbag2_storage/test_plugin.cpp

@@ -22,6 +22,7 @@
 
 #include "rosbag2_storage/serialized_bag_message.hpp"
 
+#include "test_constants.hpp"
 #include "test_plugin.hpp"
 
 TestPlugin::~TestPlugin()
@@ -79,15 +80,22 @@ rosbag2_storage::BagMetadata TestPlugin::get_metadata()
  return rosbag2_storage::BagMetadata();
 }
 
+std::string TestPlugin::get_relative_path() const
+{
+ std::cout << "\nreturning relative path\n";
+ return test_constants::DUMMY_FILEPATH;
+}
+
 uint64_t TestPlugin::get_bagfile_size() const
 {
  std::cout << "\nreturning bagfile size\n";
- return default_max_bagfile_size;
+ return test_constants::MAX_BAGFILE_SIZE;
 }
 
 std::string TestPlugin::get_storage_identifier() const
 {
- return "TestPlugin";
+ std::cout << "\nreturning storage identifier\n";
+ return test_constants::READ_WRITE_PLUGIN_IDENTIFIER;
 }
 
 PLUGINLIB_EXPORT_CLASS(TestPlugin, rosbag2_storage::storage_interfaces::ReadWriteInterface)

rosbag2_storage/test/rosbag2_storage/test_plugin.hpp

@@ -45,12 +45,11 @@ class TestPlugin : public rosbag2_storage::storage_interfaces::ReadWriteInterfac
 
  rosbag2_storage::BagMetadata get_metadata() override;
 
+ std::string get_relative_path() const override;
+
  uint64_t get_bagfile_size() const override;
 
  std::string get_storage_identifier() const override;
-
-private:
- const uint64_t default_max_bagfile_size = 0;
 };
 
 #endif // ROSBAG2_STORAGE__TEST_PLUGIN_HPP_

rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.cpp

@@ -19,6 +19,7 @@
 
 #include "pluginlib/class_list_macros.hpp"
 
+#include "test_constants.hpp"
 #include "test_read_only_plugin.hpp"
 
 TestReadOnlyPlugin::~TestReadOnlyPlugin()
@@ -54,20 +55,28 @@ std::vector<rosbag2_storage::TopicMetadata> TestReadOnlyPlugin::get_all_topics_a
  return std::vector<rosbag2_storage::TopicMetadata>();
 }
 
+std::string TestReadOnlyPlugin::get_relative_path() const
+{
+ std::cout << "\nreturning relative path\n";
+ return test_constants::DUMMY_FILEPATH;
+}
+
 rosbag2_storage::BagMetadata TestReadOnlyPlugin::get_metadata()
 {
+ std::cout << "\nreturning bag metadata\n";
  return rosbag2_storage::BagMetadata();
 }
 
 uint64_t TestReadOnlyPlugin::get_bagfile_size() const
 {
  std::cout << "\nreturning bagfile size\n";
- return default_max_bagfile_size;
+ return test_constants::MAX_BAGFILE_SIZE;
 }
 
 std::string TestReadOnlyPlugin::get_storage_identifier() const
 {
- return "TestReadOnlyPlugin";
+ std::cout << "\nreturning storage identifier\n";
+ return test_constants::READ_ONLY_PLUGIN_IDENTIFIER;
 }
 
 PLUGINLIB_EXPORT_CLASS(TestReadOnlyPlugin, rosbag2_storage::storage_interfaces::ReadOnlyInterface)

rosbag2_storage/test/rosbag2_storage/test_read_only_plugin.hpp

@@ -36,12 +36,11 @@ class TestReadOnlyPlugin : public rosbag2_storage::storage_interfaces::ReadOnlyI
 
  rosbag2_storage::BagMetadata get_metadata() override;
 
+ std::string get_relative_path() const override;
+
  uint64_t get_bagfile_size() const override;
 
  std::string get_storage_identifier() const override;
-
-private:
- const uint64_t default_max_bagfile_size = 0;
 };
 
 #endif // ROSBAG2_STORAGE__TEST_READ_ONLY_PLUGIN_HPP_