Add python message generation (#362)

azeey · web-flow · commit 73f4a45de1a1 · 2023-07-17T18:27:36.000-05:00
Signed-off-by: Addisu Z. Taddese &lt;addisu@openrobotics.org&gt;
diff --git a/.github/ci/after_make.sh b/.github/ci/after_make.sh
@@ -0,0 +1,2 @@
+# It's necessary to install the python modules for the test.
+make install
diff --git a/.github/ci/packages.apt b/.github/ci/packages.apt
@@ -6,3 +6,5 @@ libprotoc-dev
 libtinyxml2-dev
 protobuf-compiler
 ruby
+python3-pytest
+python3-protobuf
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -59,6 +59,15 @@ set(
   "gz_msgs_gen executable used in the gz_msgs_protoc CMake function.")
 mark_as_advanced(GZ_MSGS_GEN_EXECUTABLE)
 
+# Python interfaces vars
+option(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION
+      "Install python modules in standard system paths in the system"
+      OFF)
+
+option(USE_DIST_PACKAGES_FOR_PYTHON
+      "Use dist-packages instead of site-package to install python modules"
+      OFF)
+
 #============================================================================
 # Search for project-specific dependencies
 #============================================================================
@@ -87,6 +96,10 @@ set(GZ_TOOLS_VER 1)
 # Find Tinyxml2
 gz_find_package(TINYXML2 REQUIRED PRIVATE PRETTY tinyxml2)
 
+#--------------------------------------
+# Find Python
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
 #============================================================================
 # Configure the build
 #============================================================================
@@ -111,6 +124,9 @@ add_subdirectory(tools)
 # projects.
 add_subdirectory(proto)
 
+# Generate python
+add_subdirectory(python)
+
 #============================================================================
 # Create package information
 #============================================================================
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -0,0 +1,36 @@
+# Append `_configured` to the file name so it doesn't interfere with tests.
+# This happens because pytest will load the `gz.msgs` package from the build directory
+# (because there's an __init__.py file there) instead of being redirected to
+# `gz.msgs10` in the install directory, which is the intent of this `__init__.py` file.
+set(python_init_file ${PROJECT_BINARY_DIR}/python/gz/${GS_DESIGNATION}/__init__.py_configured)
+configure_file(${PROJECT_SOURCE_DIR}/python/src/__init__.py.in ${python_init_file})
+
+install(FILES ${python_init_file} DESTINATION ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/python/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR} RENAME __init__.py)
+
+if (BUILD_TESTING AND NOT WIN32)
+  set(python_tests
+    basic_TEST
+  )
+  execute_process(COMMAND "${Python3_EXECUTABLE}" -m pytest --version
+    OUTPUT_VARIABLE PYTEST_output
+    ERROR_VARIABLE  PYTEST_error
+    RESULT_VARIABLE PYTEST_result)
+  if(${PYTEST_result} EQUAL 0)
+    set(pytest_FOUND TRUE)
+  else()
+    message(WARNING "Pytest package not available: ${PYTEST_error}")
+    message(WARNING "Output: ${PYTEST_output}")
+  endif()
+
+  foreach (test ${python_tests})
+    if (pytest_FOUND)
+      add_test(NAME ${test}.py COMMAND
+        "${Python3_EXECUTABLE}" -m pytest "${CMAKE_SOURCE_DIR}/python/test/${test}.py" --junitxml "${CMAKE_BINARY_DIR}/test_results/${test}.xml")
+    else()
+      add_test(NAME ${test}.py COMMAND
+        "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/python/test/${test}.py")
+    endif()
+    set(_env_vars "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/python/")
+    set_tests_properties(${test}.py PROPERTIES ENVIRONMENT "${_env_vars}")
+  endforeach()
+endif()
diff --git a/python/src/__init__.py.in b/python/src/__init__.py.in
@@ -0,0 +1,33 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is a workaround for a limitation in out protobuf python generation
+# where a message that depends on another message will try to import the
+# corresponding python module using `gz.msgs` as the package name. However,
+# we're installing the python modules in a directory that contains the gz-msgs
+# major version number, so the import fails. This hack here overwrites the
+# entry for the unversioned module name in `sys.modules` to point to the
+# versioned module the first time a message module is loaded. Subsequent
+# imports with or without the major version number will work properly.
+
+import sys
+
+unversioned_module = "gz.msgs"
+versioned_module = "gz.msgs@PROJECT_VERSION_MAJOR@"
+if unversioned_module in sys.modules:
+    print("Looks like you are combining different versions of {}. Found {} and"
+          "{} This is not supported".format(sys.modules[unversioned_module],
+                                            sys.modules[versioned_module]))
+else:
+    sys.modules[unversioned_module] = sys.modules[versioned_module]
diff --git a/python/test/basic_TEST.py b/python/test/basic_TEST.py
@@ -0,0 +1,38 @@
+# Copyright (C) 2023 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gz.msgs10.vector3d_pb2 import Vector3d
+
+import unittest
+
+
+class BasicTest(unittest.TestCase):
+
+    def test_serialization(self):
+        msg = Vector3d()
+        msg.x = 1
+        msg.y = 2
+        msg.z = 3
+
+        serialized_msg = msg.SerializeToString()
+        self.assertGreater(len(serialized_msg), 0)
+
+        msg_from_serialized = Vector3d()
+        self.assertNotEqual(msg_from_serialized, msg)
+        msg_from_serialized.ParseFromString(serialized_msg)
+        self.assertEqual(msg_from_serialized, msg)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -19,40 +19,38 @@ if(INSTALL_GZ_MSGS_GEN_EXECUTABLE)
   install(FILES $<TARGET_FILE:gz_msgs_gen> DESTINATION ${GZ_BIN_INSTALL_DIR} RENAME ign_msgs_gen PERMISSIONS OWNER_EXECUTE)
 endif()
 
-find_package(Python3 REQUIRED COMPONENTS Interpreter)
-
 ##################################################
 # A function that calls protoc on a protobuf file
 # Options:
-#   GENERATE_RUBY       - generates ruby code for the message if specified
+#   GENERATE_PYTHON       - generates python code for the message if specified
 #   GENERATE_CPP        - generates c++ code for the message if specified
 # One value arguments:
 #   PROTO_PACKAGE       - Protobuf package the file belongs to (e.g. ".gz.msgs")
 #   PROTOC_EXEC         - Path to protoc
 #   INPUT_PROTO         - Path to the input .proto file
 #   OUTPUT_CPP_DIR      - Path where C++ files are saved
-#   OUTPUT_RUBY_DIR     - Path where Ruby files are saved
+#   OUTPUT_PYTHON_DIR     - Path where Python files are saved
 #   OUTPUT_INCLUDES     - A CMake variable name containing a list that the C++ header path should be appended to
 #   OUTPUT_CPP_HH_VAR   - A CMake variable name containing a list that the C++ header path should be appended to
 #   OUTPUT_GZ_CPP_HH_VAR - A CMake variable name containing a list that the C++ header path should be appended to
 #   OUTPUT_CPP_CC_VAR   - A Cmake variable name containing a list that the C++ source path should be appended to
-#   OUTPUT_RUBY_VAR     - A Cmake variable name containing a list that the ruby file should be apenned to
+#   OUTPUT_PYTHON_VAR     - A Cmake variable name containing a list that the python file should be appended to
 # Multi value arguments
 #   PROTO_PATH          - Passed to protoc --proto_path
 function(gz_msgs_protoc)
-  set(options GENERATE_RUBY GENERATE_CPP)
+  set(options GENERATE_PYTHON GENERATE_CPP)
   set(oneValueArgs
     PROTO_PACKAGE
     PROTOC_EXEC
     INPUT_PROTO
     OUTPUT_CPP_DIR
-    OUTPUT_RUBY_DIR
+    OUTPUT_PYTHON_DIR
     OUTPUT_INCLUDES
     OUTPUT_CPP_HH_VAR
     OUTPUT_GZ_CPP_HH_VAR
     OUTPUT_DETAIL_CPP_HH_VAR
     OUTPUT_CPP_CC_VAR
-    OUTPUT_RUBY_VAR)
+    OUTPUT_PYTHON_VAR)
   set(multiValueArgs PROTO_PATH)
 
   cmake_parse_arguments(gz_msgs_protoc "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
@@ -97,13 +95,13 @@ function(gz_msgs_protoc)
     set(${gz_msgs_protoc_OUTPUT_CPP_CC_VAR} ${${gz_msgs_protoc_OUTPUT_CPP_CC_VAR}} PARENT_SCOPE)
   endif()
 
-  if(gz_msgs_protoc_GENERATE_RUBY)
-    file(MAKE_DIRECTORY ${gz_msgs_protoc_OUTPUT_RUBY_DIR})
-    set(output_ruby "${gz_msgs_protoc_OUTPUT_RUBY_DIR}${proto_package_dir}/${FIL_WE}_pb.rb")
-    list(APPEND ${gz_msgs_protoc_OUTPUT_RUBY_VAR} ${output_ruby})
-    list(APPEND output_files ${output_ruby})
-    list(APPEND protoc_args "--ruby_out=${gz_msgs_protoc_OUTPUT_RUBY_DIR}")
-    set(${gz_msgs_protoc_OUTPUT_RUBY_VAR} ${${gz_msgs_protoc_OUTPUT_RUBY_VAR}} PARENT_SCOPE)
+  if(gz_msgs_protoc_GENERATE_PYTHON)
+    file(MAKE_DIRECTORY ${gz_msgs_protoc_OUTPUT_PYTHON_DIR})
+    # Note: Both proto2 and proto3 use the _pb2.py suffix (https://protobuf.dev/reference/python/python-generated/#invocation)
+    set(output_python "${gz_msgs_protoc_OUTPUT_PYTHON_DIR}${proto_package_dir}/${FIL_WE}_pb2.py")
+    list(APPEND ${gz_msgs_protoc_OUTPUT_PYTHON_VAR} ${output_python})
+    list(APPEND output_files ${output_python})
+    set(${gz_msgs_protoc_OUTPUT_PYTHON_VAR} ${${gz_msgs_protoc_OUTPUT_PYTHON_VAR}} PARENT_SCOPE)
   endif()
 
 
@@ -120,10 +118,10 @@ function(gz_msgs_protoc)
       --output-cpp-path "${gz_msgs_protoc_OUTPUT_CPP_DIR}")
   endif()
 
-  if(${gz_msgs_protoc_GENERATE_RUBY})
+  if(${gz_msgs_protoc_GENERATE_PYTHON})
     list(APPEND GENERATE_ARGS
-      --generate-ruby
-      --output-ruby-path "${gz_msgs_protoc_OUTPUT_RUBY_DIR}")
+      --generate-python
+      --output-python-path "${gz_msgs_protoc_OUTPUT_PYTHON_DIR}")
   endif()
 
   add_custom_command(
@@ -151,15 +149,15 @@ foreach(proto_file ${proto_files})
     PROTO_PACKAGE
       .gz.msgs
     GENERATE_CPP
-    GENERATE_RUBY
+    GENERATE_PYTHON
     INPUT_PROTO
       ${proto_file}
     PROTOC_EXEC
       protobuf::protoc
     OUTPUT_CPP_DIR
       "${PROJECT_BINARY_DIR}/include"
-    OUTPUT_RUBY_DIR
-      "${PROJECT_BINARY_DIR}/ruby"
+    OUTPUT_PYTHON_DIR
+      "${PROJECT_BINARY_DIR}/python"
     OUTPUT_INCLUDES
       gen_includes
     OUTPUT_CPP_HH_VAR
@@ -170,8 +168,8 @@ foreach(proto_file ${proto_files})
       gen_ign_headers
     OUTPUT_CPP_CC_VAR
       gen_sources
-    OUTPUT_RUBY_VAR
-      gen_ruby_scripts
+    OUTPUT_PYTHON_VAR
+      gen_python_scripts
     PROTO_PATH
       "${PROJECT_SOURCE_DIR}/proto")
 endforeach()
@@ -193,11 +191,22 @@ if(MSVC)
   add_definitions(/bigobj)
 endif()
 
-set_source_files_properties(${gen_headers} ${gen_ign_headers} ${gen_detail_headers} ${gen_sources} ${gen_ruby_scripts}
+set_source_files_properties(${gen_headers} ${gen_ign_headers} ${gen_detail_headers} ${gen_sources} ${gen_python_scripts}
   PROPERTIES GENERATED TRUE)
 
-message(STATUS "Installing Ruby messages to ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/ruby/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR}")
-install(FILES ${gen_ruby_scripts} DESTINATION ${CMAKE_INSTALL_PREFIX}/${GZ_LIB_INSTALL_DIR}/ruby/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR})
+if(USE_SYSTEM_PATHS_FOR_PYTHON_INSTALLATION)
+  if(USE_DIST_PACKAGES_FOR_PYTHON)
+    string(REPLACE "site-packages" "dist-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITELIB})
+  else()
+    # Python3_SITELIB might use dist-packages in some platforms
+    string(REPLACE "dist-packages" "site-packages" GZ_PYTHON_INSTALL_PATH ${Python3_SITELIB})
+  endif()
+else()
+  # If not a system installation, respect local paths
+  set(GZ_PYTHON_INSTALL_PATH ${GZ_LIB_INSTALL_DIR}/python)
+endif()
+message(STATUS "Installing Python messages to ${GZ_PYTHON_INSTALL_PATH}/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR}")
+install(FILES ${gen_python_scripts} DESTINATION ${GZ_PYTHON_INSTALL_PATH}/gz/${GZ_DESIGNATION}${PROJECT_VERSION_MAJOR})
 
 # Install gz/msgs
 gz_install_includes(
diff --git a/tools/gz_msgs_generate.py b/tools/gz_msgs_generate.py
@@ -35,14 +35,14 @@ def main(argv=sys.argv[1:]):
         help='Flag to indicate if C++ bindings should be generated',
         action='store_true')
     parser.add_argument(
-        '--generate-ruby',
-        help='Flag to indicate if Ruby bindings should be generated',
+        '--generate-python',
+        help='Flag to indicate if Python bindings should be generated',
         action='store_true')
     parser.add_argument(
         '--output-cpp-path',
         help='The basepath of the generated C++ files')
     parser.add_argument(
-        '--output-ruby-path',
+        '--output-python-path',
         help='The basepath of the generated C++ files')
     parser.add_argument(
         '--proto-path',
@@ -57,7 +57,7 @@ def main(argv=sys.argv[1:]):
     args = parser.parse_args(argv)
 
     for input_file in args.input_path:
-        # First generate the base cpp and ruby files
+        # First generate the base cpp and python files
         cmd = [args.protoc_exec]
 
         for pp in args.proto_path:
@@ -67,11 +67,12 @@ def main(argv=sys.argv[1:]):
             cmd += [f'--plugin=protoc-gen-ignmsgs={args.gz_generator_bin}']
             cmd += [f'--cpp_out=dllexport_decl=GZ_MSGS_VISIBLE:{args.output_cpp_path}']
             cmd += [f'--ignmsgs_out={args.output_cpp_path}']
-        if args.generate_ruby:
-            cmd += [f'--ruby_out=dllexport_decl=GZ_MSGS_VISIBLE:{args.output_ruby_path}']
+        if args.generate_python:
+            cmd += [f'--python_out={args.output_python_path}']
         cmd += [input_file]
 
         try:
+            print("cmd:", cmd)
             subprocess.check_call(cmd)
         except subprocess.CalledProcessError as e:
             print(f'Failed to execute protoc compiler: {e}')

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# It's necessary to install the python modules for the test.`
	`2`	`+make install`