Merge pull request #468 from VisionSystemsInc/new_just_pip_tools

Update new_just and to use new pip tools functions

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "2.3"
-
 services:
   wine: &wine
     build:
@@ -118,6 +116,11 @@ services:
     image: ${VSI_COMMON_DOCKER_REPO}:os_${VSI_COMMON_TEST_OS_TAG_NAME-debian_8}
     volumes:
       - <<: *vsi_common_volume
+      # Because some of the integration tests bind mount test dirs into the
+      # container, ensure the tmp dir it is using is the same tmp dir
+      - type: bind
+        source: /tmp
+        target: /tmp
     # CircleCI Updated to a kernel that supports statx, but uses an old 2017
     # server that doesn't whitelist it
     privileged: true

docker/vsi_common/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.2'
-
 services:
   vsi:
     image: vsiri/vsi_common

docs/docker/recipes/index.rst

@@ -3,5 +3,6 @@
    :caption: Contents:
 
    get-pipenv.auto
+   pip-8210.auto
 
 .. include:: README.auto

linux/ci_load.py

@@ -116,10 +116,10 @@ def _service_dict(cache_id=0, cache_loc=None):
     services = dict()
     for cache_id, cache_loc in enumerate([self.cache_repo] + self.other_repos):
       services.update(_service_dict(cache_id, cache_loc))
-    self.pull_dict = {'version': '3.5', 'services': services}
+    self.pull_dict = {'services': services}
 
     # push dictionary
-    self.push_dict = {'version': '3.5', 'services': _service_dict()}
+    self.push_dict = {'services': _service_dict()}
 
   # 2 pull images
   def pull_images(self):
@@ -150,7 +150,8 @@ def write_restore_recipe(self):
 
     doc = {}
     compose_yaml = yaml.safe_load(open(self.recipe_compose, 'r').read())
-    doc['version'] = compose_yaml['version']
+    if self.compose_version is not None:
+      doc['version'] = self.compose_version
 
     services = {}
 

linux/just_files/docker_functions.bsh

@@ -1200,7 +1200,7 @@ function get_docker_compose_version()
 
 function get_docker_stage_names()
 {
-  sed -n 's|^ *[fF][rR][oO][mM] [a-zA-Z0-9:\./_-]* [aA][sS] ||p' "${1}"
+  sed -n 's|^ *[fF][rR][oO][mM] [${}a-zA-Z0-9:\./_-]* [aA][sS] ||p' "${1}"
 }
 
 #**

linux/just_files/just_docker_functions.bsh

@@ -93,7 +93,7 @@ function docker-compose_restore_from_cache()
     Docker compose -f "${recipes_compose_file}" -f "${temp_override}" build ${recipes[@]+"${recipes[@]}"}
     # retag recipes for push
     for i in ${recipes[@]+"${recipes[@]}"}; do
-      Docker tag "vsiri/recipe:${i}" "${JUST_DOCKER_COMPOSE_CACHE_REPO}:recipe_${1}_${i}"
+      Docker tag "${VSI_RECIPE_REPO}:${i}" "${JUST_DOCKER_COMPOSE_CACHE_REPO}:recipe_${1}_${i}"
     done
   fi
 
@@ -263,14 +263,14 @@ function _dynamic_docker-compose_add_cache_from()
 # .. function:: get_docker_recipes
 #
 # :Arguments: ``$1...`` - Dockerfiles to search for recipes
-# :Parameters: ``JUST_RECIPE_REPO`` - Name of recipe image to look for. Default: ``vsiri/recipe``
+# :Parameters: ``JUST_RECIPE_REPO`` - Name of recipe image to look for. Default: ``${VSI_RECIPE_REPO}``
 # :Output: **stdout** - Newline separated list of the tags of the recipes
 #
 # Looks for recipes that are used in the Dockerfiles.
 #**
 function get_docker_recipes()
 {
-  local JUST_RECIPE_REPO="${JUST_RECIPE_REPO-vsiri/recipe}"
+  local JUST_RECIPE_REPO="${JUST_RECIPE_REPO-\\$\{VSI_RECIPE_REPO\}}"
   # / and . are the only valid character in an image name that needs regex escaping
   JUST_RECIPE_REPO="${JUST_RECIPE_REPO//./\\.}"
   # Need to escpae / for the first search
@@ -556,7 +556,7 @@ function docker_defaultify()
       # In 0.11 it was docker buildx build --invoke ...\ but I will not be supporting
       # multiple experimental versions.
       BUILDX_EXPERIMENTAL=1 Docker buildx debug --invoke /usr/bin/bash \
-        build ${build_args[@]+"${build_args[@]}"} \
+        build --progress=plain ${build_args[@]+"${build_args[@]}"} \
         -f "${dockerfile}" \
         "." # You can't use JUST_PROJECT_PREFIX+_CWD because of how terra works.
 

linux/just_files/just_pip-tools_functions.bsh

@@ -0,0 +1,175 @@
+#!/usr/bin/env false bash
+
+if [[ ${-} != *i* ]]; then
+  source_once &> /dev/null && return 0
+fi
+
+# source "${VSI_COMMON_DIR}/linux/just_files/just_singularity_functions.bsh" # Don't import here
+# source "${VSI_COMMON_DIR}/linux/just_files/docker_functions.bsh" # Don't import here
+source "${VSI_COMMON_DIR}/linux/ask_question"
+
+#*# just/plugins/just_pip-tools_functions
+
+#**
+# ============================
+# J.U.S.T. Pip-tools Functions
+# ============================
+#
+# .. default-domain:: bash
+#
+# A plugin for creating a pip-tools environment
+#
+# .. file:: just_pip-tools_functions.bsh
+#
+# .. env:: ${JUST_PROJECT_PREFIX}_PIP_TOOLS_ENVIRONMENT
+#
+# The type of environment for running in. Options are:
+#
+# * ``local`` - Runs locally in a virtual env
+# * ``docker`` - Runs in a docker
+# * ``singularity`` - Runs using singular-compose
+#
+# Default: ``local``
+#
+# .. env:: ${JUST_PROJECT_PREFIX}_PIP_TOOLS_SERVICE
+#
+# Default: ``python_cache``
+#
+# .. env:: ${JUST_PROJECT_PREFIX}_PIP_TOOLS_DIR
+#
+# Default: ``${${JUST_PROJECT_PREFIX}_CWD}/build/python`` for ``local`` or ``/venv/src`` for containers
+#**
+
+JUST_DEFAULTIFY_FUNCTIONS+=(just::defaultify::pip-tools)
+JUST_HELP_FILES+=("${BASH_SOURCE[0]}")
+
+function just::pip-tools::shell()
+{
+  local pip_tools_env="${JUST_PROJECT_PREFIX}_PIP_TOOLS_ENVIRONMENT"
+  pip_tools_env=${!pip_tools_env-local}
+  local service_name="${JUST_PROJECT_PREFIX}_PIP_TOOLS_SERVICE"
+  service_name=${!service_name-python_cache}
+  local venv_dir="${JUST_PROJECT_PREFIX}_PIP_TOOLS_DIR"
+  local source_dir="${JUST_PROJECT_PREFIX}_SOURCE_DIR"
+  source_dir="${!source_dir}"
+
+  if [ -n "${JUST_IN_CONTAINER+set}" ] || [ "${pip_tools_env}" = "local" ]; then
+    if [ -n "${JUST_IN_CONTAINER+set}" ]; then
+      venv_dir=${!venv_dir-/venv/src}
+    else
+      venv_dir=${!venv_dir-${!id_project_cwd}/build/python}
+
+      if [ ! -x "${venv_dir}/bin/python" ] && [ ! -e "${venv_dir}/Scripts/python.exe" &> /dev/null ]; then
+        justify pip-setup
+      fi
+    fi
+
+    case "${1}" in
+      pip-sync|pip-compile)
+        # Make sure pip-tools is installed (e.g. if SKIP_PIP_SYNC was used)
+        if ! command -v "${venv_dir}/bin/${1}" &> /dev/null; then
+          source "${VSI_COMMON_DIR}/linux/dir_tools.bsh"
+          local tmp_requirements
+          make_temp_path tmp_requirements
+          # Strip editables. extras are already stripped via --strip-extras
+          grep -v '^-e' "${source_dir}/requirements.txt" > "${tmp_requirements}"
+          ${venv_dir}/bin/pip install -c "${tmp_requirements}" pip-tools
+        fi
+    esac
+
+    if [ "${1}" = "pip-shell" ]; then
+      # Just run a bash shell for pip-shell
+      shift 1
+      bash --rcfile "${venv_dir}/bin/activate" ${@+"${@}"}
+    elif [ "${1}" = "pip-compile" -a "${#}" = "1" ]; then
+      "${venv_dir}/bin/pip-compile" -v --strip-extras --allow-unsafe --resolver=backtracking -o "${source_dir}/requirements.txt" "${source_dir}/requirements.in"
+      # We can't do hashes until https://github.com/VisionSystemsInc/vsi_common/issues/437 is solved
+      # exec pip-compile --strip-extras --allow-unsafe --generate-hashes --resolver=backtracking -o "${source_dir}/requirements.txt" "${source_dir}/requirements.in"
+    elif [ "${1}" = "pip-sync" -a "${#}" = "1" ]; then
+      "${venv_dir}/bin/pip-sync" -v "${source_dir}/requirements.txt"
+    else
+      bash -c 'source "'"${venv_dir}/bin/activate"'"; ${@+"${@}"}' -- ${@+"${@}"}
+    fi
+  elif [ "${pip_tools_env}" = "docker" ]; then
+    Just-docker-compose run "${service_name}" ${@+"${@}"}
+  elif [ "${pip_tools_env}" = "singularity" ]; then
+    justify singular-compose run "${service_name}" ${@+"${@}"}
+  else
+    echo "Unknown value for '${JUST_PROJECT_PREFIX}_PIP_TOOLS_ENVIRONMENT': '${pip_tools_env}'" >&2
+    return 1
+  fi
+}
+
+# function just::pip-tools::pip()
+# {
+#   just::pip-tools::shell pip ${@+"${@}"}
+# }
+
+# function just::pip-tools::python()
+# {
+#   just::pip-tools::shell python ${@+"${@}"}
+# }
+
+function just::defaultify::pip-tools()
+{
+  arg="${1}"
+  shift 1
+
+  local id_project_cwd="${JUST_PROJECT_PREFIX}_CWD"
+
+  case "${arg}" in
+    # pip) # Run pip in pip-tools managed environment
+    # pip-shell) # Start a bash shell in pip-tools managed environment
+    # pip-sync) # Run pip-sync for project
+    # pip-compile) # Run pip-compile for project
+    pip|pip-shell|pip-sync|pip-compile)
+      just::pip-tools::shell "${arg}" ${@+"${@}"}
+      extra_args=${#}
+      ;;
+
+    pip-setup) # Setup python and a venv for local pip-tools environments
+      source "${VSI_COMMON_DIR}/linux/just_files/just_install_functions.bsh"
+
+      local venv_dir="${JUST_PROJECT_PREFIX}_PIP_TOOLS_ENVIRONMENT"
+      if [ "${!venv_dir-local}" != "local" ]; then
+        echo "${RED}Error: This is for local environments only. ${venv_dir} is set to \"${!venv_dir}\"${NC}" >&2
+        return 1
+      fi
+      venv_dir="${JUST_PROJECT_PREFIX}_PIP_TOOLS_DIR"
+      venv_dir=${!venv_dir-${!id_project_cwd}/build/python}
+
+      # Setup python interpreter
+      local python_version="${JUST_PROJECT_PREFIX}_PYTHON_VERSION"
+      : ${PYTHON_VERSION=${TERRA_PYTHON_VERSION:-3.10.12}}
+      if (( ${#} )); then
+        conda-python-install --dir "${venv_dir}/.python" ${@+"${@}"}
+      else
+        conda-python-install --dir "${venv_dir}/.python" --download ${@+"${@}"}
+      fi
+      extra_args=${#}
+
+      # Setup the virtualenv
+      "${venv_dir}/.python/bin/python" -m venv "${venv_dir}"
+
+      # Setup local.env
+      local platform_bin
+      if [ "${OS-}" = "Windows_NT" ]; then
+        platform_bin=Scripts
+      else
+        platform_bin=bin
+      fi
+      local add_to_local="${add_to_local-}"
+      echo "" >&2
+      ask_question "Do you want to add \"${venv_dir}/${platform_bin}\" to your local.env automatically?" add_to_local y
+      if [ "${add_to_local}" == "1" ]; then
+        echo $'\n'"PATH=\"${venv_dir}/${platform_bin}:\${PATH}\"" >> "${!id_project_cwd}/local.env"
+      fi
+      ;;
+
+    *)
+      plugin_not_found=1
+      ;;
+  esac
+
+  return 0
+}