Pr support user doc dir

rosdoc2/verbs/build/builders/sphinx_builder.py

@@ -362,6 +362,12 @@ class SphinxBuilder(Builder):
  - If set, the documentation in this folder replaces the default documentation generated
  by rosdoc2. That is, the directory you would pass to sphinx-build as SOURCEDIR.
  Defaults to empty (or null in yaml, None in python).
+ - user_doc_dir (str) (optional)
+ - Directory (relative to the package.xml directory) where user documentation is found. If
+ documentation is in one of the standard locations (doc/ or doc/source) this is not
+ needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2
+ output, but includes this user documentation along with other items included by default.
+
  """
 
  def __init__(self, builder_name, builder_entry_dictionary, build_context):
@@ -376,6 +382,7 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context):
  self.name = self.name or self.build_context.package.name
  self.output_dir = self.output_dir or ''
  self.sphinx_sourcedir = None
+ self.user_doc_dir = None
 
  # Must check for the existence of this later, as it may not have been made yet.
  self.doxygen_xml_directory = \
@@ -392,14 +399,21 @@ def __init__(self, builder_name, builder_entry_dictionary, build_context):
  if key in ['name', 'output_dir', 'doxygen_xml_directory']:
  continue
  if key == 'sphinx_sourcedir':
- if not value:
+ if value is None:
  continue
- sphinx_sourcedir = os.path.join(configuration_file_dir, value)
- if not os.path.isdir(sphinx_sourcedir):
+ if not os.path.isdir(os.path.join(configuration_file_dir, value)):
  raise RuntimeError(
  f"Error Sphinx SOURCEDIR '{value}' does not exist relative "
  f"to '{configuration_file_path}', or is not a directory.")
- self.sphinx_sourcedir = sphinx_sourcedir
+ logger.info(f'The user specified sphinx_sourcedir as {value}')
+ self.sphinx_sourcedir = value
+ elif key == 'user_doc_dir':
+ if not os.path.isdir(os.path.join(configuration_file_dir, value)):
+ raise RuntimeError(
+ f"Error user documentation directory '{value}' does not exist relative "
+ f"to '{configuration_file_path}', or is not a directory.")
+ logger.info(f'The user specified user_doc_dir as {value}')
+ self.user_doc_dir = value
  else:
  raise RuntimeError(f"Error the Sphinx builder does not support key '{key}'")
 
@@ -475,36 +489,37 @@ def build(self, *, doc_build_folder, output_staging_directory):
  generate_standard_document_files(standard_docs, wrapped_sphinx_directory)
  logger.info(f'standard_docs: {standard_docs}')
 
- # include user documentation
- doc_directories = include_user_docs(package_xml_directory, wrapped_sphinx_directory)
- logger.info(f'doc_directories: {doc_directories}')
-
  # Check if the user provided a sphinx directory.
- sphinx_project_directory = self.sphinx_sourcedir
- if sphinx_project_directory is not None:
+ doc_directories = []
+ user_doc_dir = None
+ if self.sphinx_sourcedir is not None:
  # We do not need to check if this directory exists, as that was done in __init__.
  logger.info(
  'Note: the user provided sourcedir for Sphinx '
- f"'{sphinx_project_directory}' will be used.")
+ f"'{self.sphinx_sourcedir}' will be used.")
+ # Copy all user content, like images or documentation files, and
+ # source files to the wrapping directory
+ try:
+ shutil.copytree(
+ os.path.join(package_xml_directory, self.sphinx_sourcedir),
+ wrapped_sphinx_directory, dirs_exist_ok=True)
+ except OSError as e:
+ print(f'Failed to copy user content: {e}')
  else:
- # If the user does not supply a Sphinx sourcedir, check the standard locations.
- standard_sphinx_sourcedir = self.locate_sphinx_sourcedir_from_standard_locations()
- if standard_sphinx_sourcedir is not None:
- logger.info(
- 'Note: no sourcedir provided, but a Sphinx sourcedir located in the '
- f"standard location '{standard_sphinx_sourcedir}' and that will be used.")
- sphinx_project_directory = standard_sphinx_sourcedir
+ # include user documentation
+ if self.user_doc_dir:
+ user_doc_dir = self.user_doc_dir
  else:
- # If the user does not supply a Sphinx sourcedir, and there is no Sphinx project
- # in the conventional location, i.e. '<package dir>/doc', create a temporary
- # Sphinx project in the doc build directory to enable cross-references.
- logger.info(
- 'Note: no sourcedir provided by the user and no Sphinx sourcedir was found '
- 'in the standard locations, therefore using a default Sphinx configuration.')
-  sphinx_project_directory = os.path.join(doc_build_folder, 'default_sphinx_project')
-
- self.generate_default_project_into_directory(
-  sphinx_project_directory, python_src_directory)
+ # If the user does not supply a doc directory, check the standard locations.
+ user_doc_dir = self.locate_user_doc_dir_from_standard_locations()
+ if user_doc_dir:
+  logger.info(
+  'Note: no user_doc_dir provided, but a doc directory was located in a '
+  f'standard location "{user_doc_dir}" and that will be used.')
+ if user_doc_dir:
+ doc_directories = include_user_docs(
+  user_doc_dir, wrapped_sphinx_directory, package_xml_directory)
+ logger.info(f'doc_directories: {doc_directories}')
 
  # Collect intersphinx mapping extensions from discovered inventory files.
  inventory_files = \
@@ -534,11 +549,28 @@ def build(self, *, doc_build_folder, output_staging_directory):
  'package': self.build_context.package,
  })
 
+ # If the user did no include a conf.py, generate a default conf.py
+ if self.sphinx_sourcedir:
+ conf_py_directory = wrapped_sphinx_directory
+ elif user_doc_dir:
+ conf_py_directory = os.path.join(wrapped_sphinx_directory, user_doc_dir)
+ else:
+ conf_py_directory = None
+ if conf_py_directory and os.path.isfile(os.path.join(conf_py_directory, 'conf.py')):
+ os.rename(os.path.join(conf_py_directory, 'conf.py'),
+ os.path.join(conf_py_directory, '__conf.py'))
+ else:
+ logger.info('Note: no conf.py provided by the user, '
+ 'therefore using a default Sphinx configuration.')
+ conf_py_directory = wrapped_sphinx_directory
+ self.generate_default_project_into_directory(
+ conf_py_directory, python_src_directory)
+
  # Setup rosdoc2 Sphinx file which will include and extend the one in
  # `sphinx_project_directory`.
  self.generate_wrapping_rosdoc2_sphinx_project_into_directory(
  wrapped_sphinx_directory,
- sphinx_project_directory,
+ conf_py_directory,
  python_src_directory,
  intersphinx_mapping_extensions)
 
@@ -615,32 +647,30 @@ def build(self, *, doc_build_folder, output_staging_directory):
  # Return the directory into which Sphinx generated.
  return sphinx_output_dir
 
- def locate_sphinx_sourcedir_from_standard_locations(self):
+ def locate_user_doc_dir_from_standard_locations(self):
  """
- Return the location of a Sphinx project for the package.
+ Return the location of a user documentation for the package.
 
- If the sphinx configuration exists in a standard location, return it,
+ If the user documentation exists in a standard location, return it,
  otherwise return None. The standard locations are
- '<package.xml directory>/doc/source/conf.py' and
- '<package.xml directory>/doc/conf.py', for projects that selected
+ '<package.xml directory>/doc/source' and
+ '<package.xml directory>/doc', for projects that selected
  "separate source and build directories" when running Sphinx-quickstart and
  those that did not, respectively.
  """
  package_xml_directory = os.path.dirname(self.build_context.package.filename)
  options = [
- os.path.join(package_xml_directory, 'doc'),
- os.path.join(package_xml_directory, 'doc', 'source'),
+ os.path.join('doc', 'source'),
+ 'doc',
  ]
  for option in options:
- if os.path.isfile(os.path.join(option, 'conf.py')):
+ if os.path.isdir(os.path.join(package_xml_directory, option)):
  return option
  return None
 
  def generate_default_project_into_directory(
- self, sphinx_project_directory, python_src_directory):
- """Generate the default project configuration files."""
- os.makedirs(sphinx_project_directory, exist_ok=True)
-
+ self, conf_py_directory, python_src_directory):
+ """Generate the default project configuration files if needed."""
  package = self.build_context.package
  self.template_variables.update({
  'package': package,
@@ -652,45 +682,31 @@ def generate_default_project_into_directory(
  )),
  })
 
- with open(os.path.join(sphinx_project_directory, 'conf.py'), 'w+') as f:
+ with open(os.path.join(conf_py_directory, '__conf.py'), 'w') as f:
  f.write(default_conf_py_template.format_map(self.template_variables))
 
  def generate_wrapping_rosdoc2_sphinx_project_into_directory(
  self,
  wrapped_sphinx_directory,
- sphinx_project_directory,
+ conf_py_directory,
  python_src_directory,
  intersphinx_mapping_extensions,
  ):
  """Generate the rosdoc2 sphinx project configuration files."""
- # Generate a default index.rst
+ index_rst_path = os.path.join(wrapped_sphinx_directory, 'index.rst')
  package = self.build_context.package
- logger.info('Using a default index.rst.jinja')
- template_path = resources.files('rosdoc2.verbs.build.builders').joinpath('index.rst.jinja')
- template_jinja = template_path.read_text()
-
- index_rst = Template(template_jinja).render(self.template_variables)
-
- with open(os.path.join(wrapped_sphinx_directory, 'index.rst'), 'w+') as f:
- f.write(index_rst)
-
- # Copy all user content, like images or documentation files, and
- # source files to the wrapping directory
- #
- # If the user created an index.rst, it will overwrite our default here. Later we will
- # overwrite any user's conf.py with a wrapped version, that also includes any user's
- # conf.py variables.
- if sphinx_project_directory:
- try:
- shutil.copytree(
- os.path.abspath(sphinx_project_directory),
- os.path.abspath(wrapped_sphinx_directory),
- dirs_exist_ok=True)
+ if not os.path.isfile(index_rst_path):
+ # Generate a default index.rst
+ logger.info('Using a default index.rst.jinja')
+ template_path = resources.files(
+ 'rosdoc2.verbs.build.builders').joinpath('index.rst.jinja')
+ template_jinja = template_path.read_text()
 
- except OSError as e:
- print(f'Failed to copy user content: {e}')
+ index_rst = Template(template_jinja).render(self.template_variables)
+
+ with open(os.path.join(wrapped_sphinx_directory, 'index.rst'), 'w+') as f:
+ f.write(index_rst)
 
- package = self.build_context.package
  breathe_projects = []
  if self.doxygen_xml_directory is not None:
  breathe_projects.append(
@@ -705,7 +721,7 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory(
  'did_run_doxygen': self.doxygen_xml_directory is not None,
  'wrapped_sphinx_directory': esc_backslash(os.path.abspath(wrapped_sphinx_directory)),
  'user_conf_py_filename': esc_backslash(
- os.path.abspath(os.path.join(sphinx_project_directory, 'conf.py'))),
+ os.path.abspath(os.path.join(conf_py_directory, '__conf.py'))),
  'breathe_projects': ',\n'.join(breathe_projects) + '\n ',
  'intersphinx_mapping_extensions': ',\n '.join(intersphinx_mapping_extensions),
  'package': package,
@@ -715,5 +731,5 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory(
  'package_version_short': '.'.join(package.version.split('.')[0:2]),
  })
 
- with open(os.path.join(wrapped_sphinx_directory, 'conf.py'), 'w+') as f:
+ with open(os.path.join(wrapped_sphinx_directory, 'conf.py'), 'w') as f:
  f.write(rosdoc2_wrapping_conf_py_template.format_map(self.template_variables))

rosdoc2/verbs/build/include_user_docs.py

@@ -28,7 +28,7 @@
  :maxdepth: 1
  :glob:
 
- user_docs/*
+ {sphinx_project_directory}/*
 """
 
 subdirectory_rst_template = """\
@@ -40,52 +40,56 @@
  :maxdepth: 2
  :glob:
 
- user_docs/{name}/*
+ {sphinx_project_directory}/{name}/*
 """
 
 
-def include_user_docs(package_dir: str,
+def include_user_docs(sphinx_project_directory: str,
  output_dir: str,
+ package_xml_directory: str
  ):
  """Generate rst files for user documents."""
- logger.info(f'include_user_docs: package_dir {package_dir} output_dir {output_dir}')
- # Search the ./doc directory
- doc_dir = os.path.join(package_dir, 'doc')
- # Search /doc to insure there is at least one item of renderable documentation
+ logger.info(f'include_user_docs: sphinx_project_directory <{sphinx_project_directory}> '
+  f'output_dir <{output_dir}>')
+ user_doc_directory = os.path.join(
+  os.path.join(package_xml_directory, sphinx_project_directory))
  doc_directories = []
- for root, _, files in os.walk(doc_dir):
+ for root, _, files in os.walk(user_doc_directory):
  for file in files:
  # ensure a valid documentation file exists, directories might only contain resources.
  (_, ext) = os.path.splitext(file)
  if ext in ['.rst', '.md', '.markdown']:
  logger.debug(f'Found renderable documentation file in {root} named {file}')
- relpath = os.path.relpath(root, doc_dir)
- relpath = relpath.replace('\\', '/')
- doc_directories.append(relpath)
+ doc_directories.append(os.path.relpath(root, user_doc_directory))
  break
 
  if not doc_directories:
- logger.debug('no documentation found in /doc')
+ logger.debug(f'no documentation found in {sphinx_project_directory}')
  return doc_directories
 
- logger.info(f'Documentation found in /doc in directories {doc_directories}')
+ logger.info(f'Documentation found in directories {doc_directories}')
  # At this point we know that there are some directories that have documentation in them under
  # /doc, but we do not know which ones might also be needed for images or includes. So we copy
  # everything to the output directory.
+ logger.info(f'Copying {os.path.join(package_xml_directory, sphinx_project_directory)} to '
+ f'{os.path.join(output_dir, sphinx_project_directory)}')
  shutil.copytree(
- os.path.abspath(doc_dir),
- os.path.abspath(os.path.join(output_dir, 'user_docs')),
+ os.path.join(package_xml_directory, sphinx_project_directory),
+ os.path.join(output_dir, sphinx_project_directory),
  dirs_exist_ok=True)
 
- toc_content = documentation_rst_template
+ toc_content = documentation_rst_template.format_map(
+ {'sphinx_project_directory': sphinx_project_directory})
  # generate a glob rst entry for each directory with documents
  for relpath in doc_directories:
  # directories that will be explicitly listed in index.rst
  if relpath == '.':
  continue
  docname = 'user_docs_' + slugify(relpath) # This is the name that sphinx uses
  content = subdirectory_rst_template.format_map(
- {'name': relpath, 'name_underline': '=' * len(relpath)})
+ {'name': relpath,
+ 'name_underline': '=' * len(relpath),
+ 'sphinx_project_directory': sphinx_project_directory})
  sub_path = os.path.join(output_dir, docname + '.rst')
  with open(sub_path, 'w+') as f:
  f.write(content)

rosdoc2/verbs/build/inspect_package_for_settings.py

@@ -80,6 +80,12 @@
  ## doc/source/ folder will still be included by default, along with other relevant package
  ## information.
  # sphinx_sourcedir: null,
+ ## Directory (relative to the package.xml directory) where user documentation is found. If
+ ## documentation is in one of the standard locations (doc/ or doc/source) this is not
+ ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2
+ ## output, but includes this user documentation along with other items included by default
+ ## by rosdoc2.
+ # user_doc_dir: 'doc'
  }}
 """
 

test/packages/default_yaml/rosdoc2.yaml

@@ -59,4 +59,10 @@ builders:
  ## doc/source/ folder will still be included by default, along with other relevant package
  ## information.
  sphinx_sourcedir: null,
+ ## Directory (relative to the package.xml directory) where user documentation is found. If
+ ## documentation is in one of the standard locations (doc/ or doc/source) this is not
+ ## needed. Unlike sphinx_sourcedir, specifying this does not override the standard rosdoc2
+ ## output, but includes this user documentation along with other items included by default
+ ## by rosdoc2.
+ user_doc_dir: 'doc'
  }

test/packages/empty_doc_dir/doc/dummy.stuff

@@ -0,0 +1 @@
+This file does not contain any documentation.

test/packages/empty_doc_dir/package.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="3">
+ <name>empty_doc_dir</name>
+ <version>0.1.2</version>
+ <description>Package with an empty doc directory</description>
+ <maintainer email="[email protected]">Some One</maintainer>
+ <license>Apache License 2.0</license>
+</package>

test/packages/false_python/docs/moredocs/more1.rst

@@ -0,0 +1,4 @@
+This is some more documentation
+===============================
+
+blah, blah.

test/packages/false_python/docs/somedocs.md

@@ -0,0 +1,3 @@
+# This is documentation
+
+blah, blah

test/packages/false_python/package.xml

@@ -8,5 +8,6 @@
  <license>Apache-2.0</license>
  <export>
  <build_type>ament_python</build_type>
+ <rosdoc2>rosdoc2.yaml</rosdoc2>
  </export>
 </package>