Add python message generation

.github/ci/after_make.sh

@@ -0,0 +1,2 @@
+# It's necessary to install the python modules for the test.
+make install

.github/ci/packages.apt

@@ -6,3 +6,5 @@ libprotoc-dev
 libtinyxml2-dev
 protobuf-compiler
 ruby
+python3-pytest
+python3-protobuf

CMakeLists.txt

@@ -59,6 +59,15 @@ set(
  "gz_msgs_gen executable used in the gz_msgs_protoc CMake function.")
 mark_as_advanced(GZ_MSGS_GEN_EXECUTABLE)
 
+# Python interfaces vars
+option(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION
+ "Install python modules in standard system paths in the system"
+ OFF)
+
+option(USE_DIST_PACKAGES_FOR_PYTHON
+ "Use dist-packages instead of site-package to install python modules"
+ OFF)
+
 #============================================================================
 # Search for project-specific dependencies
 #============================================================================
@@ -87,6 +96,10 @@ set(GZ_TOOLS_VER 1)
 # Find Tinyxml2
 gz_find_package(TINYXML2 REQUIRED PRIVATE PRETTY tinyxml2)
 
+#--------------------------------------
+# Find Python
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
 #============================================================================
 # Configure the build
 #============================================================================
@@ -111,6 +124,9 @@ add_subdirectory(tools)
 # projects.
 add_subdirectory(proto)
 
+# Generate python
+add_subdirectory(python)
+
 #============================================================================
 # Create package information
 #============================================================================

python/CMakeLists.txt

@@ -0,0 +1,36 @@
+# Append `_configured` to the file name so it doesn't interfere with tests.
+# This happens because pytest will load the `gz.msgs` package from the build directory
+# (because there's an __init__.py file there) instead of being redirected to
+# `gz.msgs10` in the install directory, which is the intent of this `__init__.py` file.
+set(python_init_file ${PROJECT_BINARY_DIR}/python/gz/${GS_DESIGNATION}/__init__.py_configured)
+configure_file(${PROJECT_SOURCE_DIR}/python/src/__init__.py.in ${python_init_file})
+
+install(FILES ${python_init_file} DESTINATION ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/python/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR} RENAME __init__.py)
+
+if (BUILD_TESTING AND NOT WIN32)
+ set(python_tests
+ basic_TEST
+ )
+ execute_process(COMMAND "${Python3_EXECUTABLE}" -m pytest --version
+ OUTPUT_VARIABLE PYTEST_output
+ ERROR_VARIABLE PYTEST_error
+ RESULT_VARIABLE PYTEST_result)
+ if(${PYTEST_result} EQUAL 0)
+ set(pytest_FOUND TRUE)
+ else()
+ message(WARNING "Pytest package not available: ${PYTEST_error}")
+ message(WARNING "Output: ${PYTEST_output}")
+ endif()
+
+ foreach (test ${python_tests})
+ if (pytest_FOUND)
+ add_test(NAME ${test}.py COMMAND
+ "${Python3_EXECUTABLE}" -m pytest "${CMAKE_SOURCE_DIR}/python/test/${test}.py" --junitxml "${CMAKE_BINARY_DIR}/test_results/${test}.xml")
+ else()
+ add_test(NAME ${test}.py COMMAND
+ "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/python/test/${test}.py")
+ endif()
+ set(_env_vars "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/python/")
+ set_tests_properties(${test}.py PROPERTIES ENVIRONMENT "${_env_vars}")
+ endforeach()
+endif()

python/src/__init__.py.in

@@ -0,0 +1,33 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is a workaround for a limitation in out protobuf python generation
+# where a message that depends on another message will try to import the
+# corresponding python module using `gz.msgs` as the package name. However,
+# we're installing the python modules in a directory that contains the gz-msgs
+# major version number, so the import fails. This hack here overwrites the
+# entry for the unversioned module name in `sys.modules` to point to the
+# versioned module the first time a message module is loaded. Subsequent
+# imports with or without the major version number will work properly.
+
+import sys
+
+unversioned_module = "gz.msgs"
+versioned_module = "gz.msgs@PROJECT_VERSION_MAJOR@"
+if unversioned_module in sys.modules:
+ print("Looks like you are combining different versions of {}. Found {} and"
+ "{} This is not supported".format(sys.modules[unversioned_module],
+ sys.modules[versioned_module]))
+else:
+ sys.modules[unversioned_module] = sys.modules[versioned_module]

python/test/basic_TEST.py

@@ -0,0 +1,38 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gz.msgs10.vector3d_pb2 import Vector3d
+
+import unittest
+
+
+class BasicTest(unittest.TestCase):
+
+ def test_serialization(self):
+ msg = Vector3d()
+ msg.x = 1
+ msg.y = 2
+ msg.z = 3
+
+ serialized_msg = msg.SerializeToString()
+ self.assertGreater(len(serialized_msg), 0)
+
+ msg_from_serialized = Vector3d()
+ self.assertNotEqual(msg_from_serialized, msg)
+ msg_from_serialized.ParseFromString(serialized_msg)
+ self.assertEqual(msg_from_serialized, msg)
+
+
+if __name__ == '__main__':
+ unittest.main()

src/CMakeLists.txt

@@ -19,40 +19,38 @@ if(INSTALL_GZ_MSGS_GEN_EXECUTABLE)
  install(FILES $<TARGET_FILE:gz_msgs_gen> DESTINATION ${GZ_BIN_INSTALL_DIR} RENAME ign_msgs_gen PERMISSIONS OWNER_EXECUTE)
 endif()
 
-find_package(Python3 REQUIRED COMPONENTS Interpreter)
-
 ##################################################
 # A function that calls protoc on a protobuf file
 # Options:
-# GENERATE_RUBY - generates ruby code for the message if specified
+# GENERATE_PYTHON - generates python code for the message if specified
 # GENERATE_CPP - generates c++ code for the message if specified
 # One value arguments:
 # PROTO_PACKAGE - Protobuf package the file belongs to (e.g. ".gz.msgs")
 # PROTOC_EXEC - Path to protoc
 # INPUT_PROTO - Path to the input .proto file
 # OUTPUT_CPP_DIR - Path where C++ files are saved
-# OUTPUT_RUBY_DIR - Path where Ruby files are saved
+# OUTPUT_PYTHON_DIR - Path where Python files are saved
 # OUTPUT_INCLUDES - A CMake variable name containing a list that the C++ header path should be appended to
 # OUTPUT_CPP_HH_VAR - A CMake variable name containing a list that the C++ header path should be appended to
 # OUTPUT_GZ_CPP_HH_VAR - A CMake variable name containing a list that the C++ header path should be appended to
 # OUTPUT_CPP_CC_VAR - A Cmake variable name containing a list that the C++ source path should be appended to
-# OUTPUT_RUBY_VAR - A Cmake variable name containing a list that the ruby file should be apenned to
+# OUTPUT_PYTHON_VAR - A Cmake variable name containing a list that the python file should be appended to
 # Multi value arguments
 # PROTO_PATH - Passed to protoc --proto_path
 function(gz_msgs_protoc)
- set(options GENERATE_RUBY GENERATE_CPP)
+ set(options GENERATE_PYTHON GENERATE_CPP)
  set(oneValueArgs
  PROTO_PACKAGE
  PROTOC_EXEC
  INPUT_PROTO
  OUTPUT_CPP_DIR
- OUTPUT_RUBY_DIR
+ OUTPUT_PYTHON_DIR
  OUTPUT_INCLUDES
  OUTPUT_CPP_HH_VAR
  OUTPUT_GZ_CPP_HH_VAR
  OUTPUT_DETAIL_CPP_HH_VAR
  OUTPUT_CPP_CC_VAR
- OUTPUT_RUBY_VAR)
+ OUTPUT_PYTHON_VAR)
  set(multiValueArgs PROTO_PATH)
 
  cmake_parse_arguments(gz_msgs_protoc "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
@@ -97,13 +95,13 @@ function(gz_msgs_protoc)
  set(${gz_msgs_protoc_OUTPUT_CPP_CC_VAR} ${${gz_msgs_protoc_OUTPUT_CPP_CC_VAR}} PARENT_SCOPE)
  endif()
 
- if(gz_msgs_protoc_GENERATE_RUBY)
- file(MAKE_DIRECTORY ${gz_msgs_protoc_OUTPUT_RUBY_DIR})
- set(output_ruby "${gz_msgs_protoc_OUTPUT_RUBY_DIR}${proto_package_dir}/${FIL_WE}_pb.rb")
- list(APPEND ${gz_msgs_protoc_OUTPUT_RUBY_VAR} ${output_ruby})
- list(APPEND output_files ${output_ruby})
- list(APPEND protoc_args "--ruby_out=${gz_msgs_protoc_OUTPUT_RUBY_DIR}")
- set(${gz_msgs_protoc_OUTPUT_RUBY_VAR} ${${gz_msgs_protoc_OUTPUT_RUBY_VAR}} PARENT_SCOPE)
+ if(gz_msgs_protoc_GENERATE_PYTHON)
+ file(MAKE_DIRECTORY ${gz_msgs_protoc_OUTPUT_PYTHON_DIR})
+ # Note: Both proto2 and proto3 use the _pb2.py suffix (https://protobuf.dev/reference/python/python-generated/#invocation)
+ set(output_python "${gz_msgs_protoc_OUTPUT_PYTHON_DIR}${proto_package_dir}/${FIL_WE}_pb2.py")
+ list(APPEND ${gz_msgs_protoc_OUTPUT_PYTHON_VAR} ${output_python})
+ list(APPEND output_files ${output_python})
+ set(${gz_msgs_protoc_OUTPUT_PYTHON_VAR} ${${gz_msgs_protoc_OUTPUT_PYTHON_VAR}} PARENT_SCOPE)
  endif()
 
 
@@ -120,10 +118,10 @@ function(gz_msgs_protoc)
  --output-cpp-path "${gz_msgs_protoc_OUTPUT_CPP_DIR}")
  endif()
 
- if(${gz_msgs_protoc_GENERATE_RUBY})
+ if(${gz_msgs_protoc_GENERATE_PYTHON})
  list(APPEND GENERATE_ARGS
- --generate-ruby
- --output-ruby-path "${gz_msgs_protoc_OUTPUT_RUBY_DIR}")
+ --generate-python
+ --output-python-path "${gz_msgs_protoc_OUTPUT_PYTHON_DIR}")
  endif()
 
  add_custom_command(
@@ -151,15 +149,15 @@ foreach(proto_file ${proto_files})
  PROTO_PACKAGE
  .gz.msgs
  GENERATE_CPP
- GENERATE_RUBY
+ GENERATE_PYTHON
  INPUT_PROTO
  ${proto_file}
  PROTOC_EXEC
  protobuf::protoc
  OUTPUT_CPP_DIR
  "${PROJECT_BINARY_DIR}/include"
- OUTPUT_RUBY_DIR
- "${PROJECT_BINARY_DIR}/ruby"
+ OUTPUT_PYTHON_DIR
+ "${PROJECT_BINARY_DIR}/python"
  OUTPUT_INCLUDES
  gen_includes
  OUTPUT_CPP_HH_VAR
@@ -170,8 +168,8 @@ foreach(proto_file ${proto_files})
  gen_ign_headers
  OUTPUT_CPP_CC_VAR
  gen_sources
- OUTPUT_RUBY_VAR
- gen_ruby_scripts
+ OUTPUT_PYTHON_VAR
+ gen_python_scripts
  PROTO_PATH
  "${PROJECT_SOURCE_DIR}/proto")
 endforeach()
@@ -193,11 +191,22 @@ if(MSVC)
  add_definitions(/bigobj)
 endif()
 
-set_source_files_properties(${gen_headers} ${gen_ign_headers} ${gen_detail_headers} ${gen_sources} ${gen_ruby_scripts}
+set_source_files_properties(${gen_headers} ${gen_ign_headers} ${gen_detail_headers} ${gen_sources} ${gen_python_scripts}
  PROPERTIES GENERATED TRUE)
 
-message(STATUS "Installing Ruby messages to ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/ruby/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR}")
-install(FILES ${gen_ruby_scripts} DESTINATION ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/ruby/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR})
+if(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION)
+ if(USE_DIST_PACKAGES_FOR_PYTHON)
+ string(REPLACE "site-packages" "dist-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITELIB})
+ else()
+ # Python3_SITELIB might use dist-packages in some platforms
+ string(REPLACE "dist-packages" "site-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITELIB})
+ endif()
+else()
+ # If not a system installation, respect local paths
+ set(GZ_PYTHON_INSTALL_PATH ${GZ_LIB_INSTALL_DIR}/python)
+endif()
+message(STATUS "Installing Python messages to ${GZ_PYTHON_INSTALL_PATH}/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR}")
+install(FILES ${gen_python_scripts} DESTINATION ${GZ_PYTHON_INSTALL_PATH}/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR})
 
 # Install gz/msgs
 gz_install_includes(

tools/gz_msgs_generate.py

@@ -35,14 +35,14 @@ def main(argv=sys.argv[1:]):
  help='Flag to indicate if C++ bindings should be generated',
  action='store_true')
  parser.add_argument(
- '--generate-ruby',
- help='Flag to indicate if Ruby bindings should be generated',
+ '--generate-python',
+ help='Flag to indicate if Python bindings should be generated',
  action='store_true')
  parser.add_argument(
  '--output-cpp-path',
  help='The basepath of the generated C++ files')
  parser.add_argument(
- '--output-ruby-path',
+ '--output-python-path',
  help='The basepath of the generated C++ files')
  parser.add_argument(
  '--proto-path',
@@ -57,7 +57,7 @@ def main(argv=sys.argv[1:]):
  args = parser.parse_args(argv)
 
  for input_file in args.input_path:
- # First generate the base cpp and ruby files
+ # First generate the base cpp and python files
  cmd = [args.protoc_exec]
 
  for pp in args.proto_path:
@@ -67,11 +67,12 @@ def main(argv=sys.argv[1:]):
  cmd += [f'--plugin=protoc-gen-ignmsgs={args.gz_generator_bin}']
  cmd += [f'--cpp_out=dllexport_decl=GZ_MSGS_VISIBLE:{args.output_cpp_path}']
  cmd += [f'--ignmsgs_out={args.output_cpp_path}']
- if args.generate_ruby:
- cmd += [f'--ruby_out=dllexport_decl=GZ_MSGS_VISIBLE:{args.output_ruby_path}']
+ if args.generate_python:
+ cmd += [f'--python_out={args.output_python_path}']
  cmd += [input_file]
 
  try:
+ print("cmd:", cmd)
  subprocess.check_call(cmd)
  except subprocess.CalledProcessError as e:
  print(f'Failed to execute protoc compiler: {e}')