[WIP] Enable true cross compilation workflow

.gitignore

@@ -5,9 +5,9 @@ __pycache__/
 .tox/
 /.coverage
 /coverage.xml
-
 .idea/
-
 sysroot/
-
 .DS_Store
+*.ipynb
+*.tags
+*.tags1

ros_cross_compile/builders.py

@@ -69,3 +69,36 @@ def __call__(
         data_collector: DataCollector
     ):
         run_emulated_docker_build(docker_client, platform, ros_workspace_dir)
+
+
+def run_cross_compile_docker_build(
+    docker_client: DockerClient,
+    platform: Platform,
+    workspace_path: Path,
+) -> None:
+    docker_client.build_image(
+        dockerfile_name='build.Dockerfile',
+        tag=platform.build_image_tag,
+    )
+
+    docker_client.run_container(
+        image_name=platform.build_image_tag,
+        environment={
+            'OWNER_USER': str(os.getuid()),
+            'ROS_DISTRO': platform.ros_distro,
+            'TARGET_ARCH': platform.arch,
+        },
+        volumes={
+            workspace_path: '/ros_ws',
+        }
+    )
+
+
+class CrossCompileBuild(PipelineStage):
+    def __init__(self):
+        super().__init__('cross_compile_build')
+
+    def __call__(self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path,
+                 options: PipelineStageConfigOptions,
+                 data_collector: DataCollector):
+        run_cross_compile_docker_build(docker_client, platform, ros_workspace_dir)

ros_cross_compile/dependencies.py

@@ -23,7 +23,7 @@
 from ros_cross_compile.pipeline_stages import PipelineStage
 from ros_cross_compile.pipeline_stages import PipelineStageOptions
 from ros_cross_compile.platform import Platform
-from ros_cross_compile.sysroot_creator import build_internals_dir
+from ros_cross_compile.sysroot_creator import build_internals_dir, rosdep_install_script
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger('Rosdep Gatherer')
@@ -34,9 +34,6 @@
 _IMG_NAME = 'ros_cross_compile:rosdep'
 
 
-def rosdep_install_script(platform: Platform) -> Path:
-    """Construct relative path of the script that installs rosdeps into the sysroot image."""
-    return build_internals_dir(platform) / 'install_rosdeps.sh'
 
 
 def gather_rosdeps(

ros_cross_compile/docker/build.Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:focal
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Common for all
+RUN apt-get update && apt-get install --no-install-recommends -q -y \
+    build-essential \
+    cmake \
+    python3-pip \
+    wget
+
+RUN pip3 install colcon-common-extensions colcon-mixin
+
+# Specific at the end (layer sharing)
+RUN apt-get update && apt-get install --no-install-recommends -q -y \
+    gcc-aarch64-linux-gnu \
+    g++-aarch64-linux-gnu
+
+RUN apt-get update && apt-get install -q -y --no-install-recommends rsync
+
+RUN pip3 install lark-parser numpy
+
+# Fast and small, no optimization necessary
+COPY mixins/ /mixins/
+COPY build_workspace.sh /root
+COPY toolchains/ /toolchains/
+WORKDIR /ros_ws
+ENTRYPOINT ["/root/build_workspace.sh"]

ros_cross_compile/docker/build_workspace.sh

@@ -7,16 +7,29 @@ cleanup() {
 
 trap 'cleanup' EXIT
 
-mkdir -p /opt/ros/"${ROS_DISTRO}"
-touch /opt/ros/"${ROS_DISTRO}"/setup.bash
+export SYSROOT=/ros_ws/cc_internals/sysroot
+export ROS_WS_INSTALL_PATH=/ros_ws/install_${TARGET_ARCH}
+export ROS_WS_BUILD_PATH=/ros_ws/build_${TARGET_ARCH}
+
+rosdir=${SYSROOT}/opt/ros/${ROS_DISTRO}
+
+# It's possible that the workspace does not require ROS binary dependencies
+# so this could not have been created. Instead of checking, lazily touch it
+mkdir -p ${rosdir}
+touch ${rosdir}/setup.bash
+
+export TRIPLE=aarch64-linux-gnu
+rsync -a ${SYSROOT}/usr/lib/${TRIPLE}/ /usr/lib/${TRIPLE}/
+rsync -a ${SYSROOT}/usr/include/ /usr/include/
 
 set +ux
 # shellcheck source=/dev/null
-source /opt/ros/"${ROS_DISTRO}"/setup.bash
+source ${rosdir}/setup.bash
 set -ux
-colcon build --mixin "${TARGET_ARCH}"-docker \
-  --build-base build_"${TARGET_ARCH}" \
-  --install-base install_"${TARGET_ARCH}"
-
+# export MAKEFLAGS="-j1"
+colcon build \
+  --build-base ${ROS_WS_BUILD_PATH} \
+  --install-base ${ROS_WS_INSTALL_PATH} \
+  --cmake-args -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_TOOLCHAIN_FILE=/toolchains/${TARGET_ARCH}-gnu.cmake --no-warn-unused-cli
 # Runs user-provided post-build logic (file is present and empty if it wasn't specified)
 /user-custom-post-build

ros_cross_compile/docker/gather_rosdeps.sh

@@ -49,7 +49,7 @@ grep "apt-get install -y" /tmp/all-deps.sh > /tmp/apt-deps.sh || true
 # awk notes:
 #  "apt-get", "install", "-y", package_name is the fourth column
 #  ORS=' ' makes the output space-separated instead of newline-separated output
-echo "apt-get install -y $(awk '{print $4}' ORS=' ' < /tmp/apt-deps.sh)" >> "${OUT_PATH}"
+echo "apt-get install -y --no-install-recommends $(awk '{print $4}' ORS=' ' < /tmp/apt-deps.sh)" >> "${OUT_PATH}"
 
 chmod +x "${OUT_PATH}"
 chown -R "${OWNER_USER}" "${out_dir}"

ros_cross_compile/docker/runtime.Dockerfile

@@ -8,6 +8,8 @@ FROM $BASE_IMAGE
 
 WORKDIR /ros_ws
 
+RUN apt-get update && apt-get install -q -y --no-install-recommends less vim
+
 ARG INSTALL_PATH
 COPY $INSTALL_PATH/ install
 

ros_cross_compile/docker/sysroot.Dockerfile

@@ -5,71 +5,33 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 
-ARG ROS_VERSION
-
 SHELL ["/bin/bash", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
 
+# Grab the qemu binaries, if any, that were placed in the build context for us
 COPY bin/* /usr/bin/
 
-# Set timezone
-RUN echo 'Etc/UTC' > /etc/timezone && \
-    ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
-
-RUN apt-get update && apt-get install --no-install-recommends -y \
-        tzdata \
-        locales \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set locale
-RUN echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen && \
-    locale-gen && \
-    update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LC_ALL C.UTF-8
-
-# Add the ros apt repo
+# # Add the ros apt repo
 RUN apt-get update && apt-get install --no-install-recommends -y \
-        dirmngr \
         gnupg2 \
         lsb-release \
     && rm -rf /var/lib/apt/lists/*
 RUN apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' \
     --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654
 
-RUN echo "deb http://packages.ros.org/${ROS_VERSION}/ubuntu `lsb_release -cs` main" \
-    > /etc/apt/sources.list.d/${ROS_VERSION}-latest.list
+RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -cs` main" \
+    >> /etc/apt/sources.list.d/ros-latest.list
+RUN echo "deb http://packages.ros.org/ros2/ubuntu `lsb_release -cs` main" \
+    >> /etc/apt/sources.list.d/ros-latest.list
 
 # ROS dependencies
 RUN apt-get update && apt-get install --no-install-recommends -y \
-      build-essential \
-      cmake \
-      python3-colcon-common-extensions \
-      python3-colcon-mixin \
-      python3-dev \
       python3-pip \
+      libssl-dev \
+      symlinks \
     && rm -rf /var/lib/apt/lists/*
 
-RUN python3 -m pip install -U \
-    setuptools
-
-# Install some pip packages needed for testing ROS 2
-RUN if [[ "${ROS_VERSION}" == "ros2" ]]; then \
-    python3 -m pip install -U \
-    flake8 \
-    flake8-blind-except \
-    flake8-builtins \
-    flake8-class-newline \
-    flake8-comprehensions \
-    flake8-deprecated \
-    flake8-docstrings \
-    flake8-import-order \
-    flake8-quotes \
-    pytest-repeat \
-    pytest-rerunfailures \
-    pytest \
-    pytest-cov \
-    pytest-runner \
-  ; fi
+ARG ROS_VERSION
 
 # Install Fast-RTPS dependencies for ROS 2
 RUN if [[ "${ROS_VERSION}" == "ros2" ]]; then \
@@ -86,13 +48,17 @@ RUN chmod +x ./user-custom-setup && \
     ./user-custom-setup && \
     rm -rf /var/lib/apt/lists/*
 
+ARG DEPENDENCY_SCRIPT
 # Use generated rosdep installation script
-COPY install_rosdeps.sh .
-RUN chmod +x install_rosdeps.sh
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
-    ./install_rosdeps.sh && \
+COPY ${DEPENDENCY_SCRIPT} .
+RUN chmod +x ${DEPENDENCY_SCRIPT}
+RUN apt-get update && \
+    ./${DEPENDENCY_SCRIPT} && \
     rm -rf /var/lib/apt/lists/*
 
+# Make all absolute symlinks in the filesystem relative, so that we can use it for cross-compilation
+RUN symlinks -rc /
+
 # Set up build tools for the workspace
 COPY mixins/ mixins/
 RUN colcon mixin add cc_mixin file://$(pwd)/mixins/index.yaml && colcon mixin update cc_mixin

ros_cross_compile/docker_client.py

@@ -11,21 +11,41 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import io
 import logging
 from pathlib import Path
+import tarfile
 from typing import Dict
 from typing import Optional
 
 import docker
 from docker.utils import kwargs_from_env as docker_kwargs_from_env
 
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger('Docker Client')
 
 DEFAULT_COLCON_DEFAULTS_FILE = 'defaults.yaml'
 
 
+class GeneratorStream(io.RawIOBase):
+    def __init__(self, generator):
+        self.leftover = None
+        self.generator = generator
+
+    def readable(self):
+        return True
+
+    def readinto(self, b):
+        try:
+            length = len(b)  # : We're supposed to return at most this much
+            chunk = self.leftover or next(self.generator)
+            output, self.leftover = chunk[:length], chunk[length:]
+            b[:len(output)] = output
+            return len(output)
+        except StopIteration:
+            return 0  # : Indicate EOF
+
+
 class DockerClient:
     """Simplified Docker API for this package's usage patterns."""
 
@@ -65,6 +85,7 @@ def build_image(
         """
         # Use low-level API to expose logs for image building
         docker_api = docker.APIClient(**docker_kwargs_from_env())
+        logger.info('Sending context to Docker client')
         log_generator = docker_api.build(
             path=str(dockerfile_dir) if dockerfile_dir else self._default_docker_dir,
             dockerfile=dockerfile_name,
@@ -119,6 +140,7 @@ def run_container(
         # Note that the `run` kwarg `stream` is not available
         # in the version of dockerpy that we are using, so we must detach to live-stream logs
         # Do not `remove` so that the container can be queried for its exit code after finishing
+        logger.info("Running docker container of image {}".format(image_name))
         container = self._client.containers.run(
             image=image_name,
             name=container_name,
@@ -146,3 +168,10 @@ def run_container(
 
     def get_image_size(self, img_name: str) -> int:
         return self._client.images.get(img_name).attrs['Size']
+
+    def export_image_filesystem(self, image_tag: str):
+        container = self._client.containers.run(image=image_tag, detach=True)
+        export_generator = container.export()
+        stream = io.BufferedReader(GeneratorStream(export_generator))
+        tar = tarfile.open(fileobj=stream, mode='r|*')
+        return tar