.gitlab-ci.yml

# =========================================================================
#   Program:   S1Processor
#
#   Copyright 2017-2024 (c) CNES. All rights reserved.
#
#   This file is part of S1Tiling project
#       https://gitlab.orfeo-toolbox.org/s1-tiling/s1tiling
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# =========================================================================
#
# Authors: Thierry KOLECK (CNES)
#          Luc HERMITTE (CS Group)
#
# =========================================================================

variables:
    BUILD_IMAGE_REGISTRY: $CI_REGISTRY/s1-tiling
    LC_ALL: C.UTF-8
    # Set pip's cache inside the project directory since we can only cache local items
    # PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
    # See Issue#86: https://gitlab.orfeo-toolbox.org/s1-tiling/s1tiling/-/issues/86
    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: 0
    S1TD_VERSION: "1.3.8"  # S1Tiling Dockers version
    OTB9_VERSION: "9.0.0"
    OS: "ubuntu"

#
# Tests and documentation
#
.common:
    variables:
      BASE_FULL_VERSION: ${S1TD_VERSION}-${OS}${OS_VERSION}-otb${OTB_VERSION}
    image: ${BUILD_IMAGE_REGISTRY}/s1tiling-dockers/s1tiling-dev:${BASE_FULL_VERSION}
    # cache:
    #     key: packages
    #     paths:
    #     # - $CI_PROJECT_DIR/.cache/pip
    #     # - S1TilingEnv
    #     # - baseline

    before_script:
    # - mkdir -p "${PIP_CACHE_DIR}"
    ## doc generation requires S1Tiling through autodoc
    # => We need to be able to install S1Tiling
    # * Update system
    - apt-get -y update
    - apt-get -y upgrade
    - python3 --version
    # * Check Python virtenv is correctly set even without explicit activation
    - echo "Check VIRTUAL_ENV=${VIRTUAL_ENV}"
    - python -c "import numpy ; print(numpy.__version__)"
    # * Check OTB variables are correctly set even without the ". otbenv.profile"
    - echo "Check OTB_APPLICATION_PATH= ${OTB_APPLICATION_PATH}"
    - which otbcli_ResetMargin
    # * Check GDAL and its bindings are correctly installed
    - gdal-config --version
    - echo $LD_LIBRARY_PATH
    - find / -name "libgdal.so*"
    - ldd ${S1TILING_VENV}/lib/python*/site-packages/osgeo/_gdal.cpython-3*-x86_64-linux-gnu.so
    - python -c "from osgeo import gdal ; print(gdal.__version__)"
    - ls "${OTB_INSTALL_DIRNAME}"
    # * Install S1Tiling
    - echo "installing .[${INSTALL_TYPE}]"
    - python3 -m pip install --upgrade pip
    - python3 -m pip install .[${INSTALL_TYPE}]

# TODO in .common-dev/doc: reuse what have been done in .common up to
# "pip install ."
.common-doc:
    variables:
        INSTALL_TYPE: docs
    extends: .common

.common-dev:
    variables:
        INSTALL_TYPE: dev
    extends: .common

# Package
# TODO: Reuse installation done in .common-dev
package9:
  extends: .common-dev
  variables:
    OS: ubuntu
    OS_VERSION: "22.04"
    OTB_VERSION: "${OTB9_VERSION}"
  stage: build
  script:
    - python3 setup.py sdist bdist_wheel
  artifacts:
    paths:
      - dist/

# Run tests
# TODO: Reuse installation done in .common-dev
.pytest:
    resource_group: raoul_is_really_constrainst
    variables:
        KUBERNETES_CPU_REQUEST: 4
        KUBERNETES_MEMORY_REQUEST: 4Gi
    extends: .common-dev
    stage: test
    script:
    - test -d baseline || mkdir baseline
    - rclone -v -L sync s3_otb:s1-tiling/baseline baseline || echo "Oups| no baseline/ on S3"
    - ls baseline
    # - find baseline/expected -type f -exec md5sum {} \;
    - (cd baseline && find . -type f -exec md5sum {} \;)
    - python3 -m pytest --baselinedir=$(readlink -f baseline) --demdir=$(readlink -f baseline/MNT/SRTM_30_hgt) --ram=2048 -o log_cli_level=info -o log_cli=true --junitxml=report.xml
    artifacts:
        when: always
        reports:
            junit: report.xml

# Test the documentation compiles
# TODO: Reuse installation done in .common-doc
.test-doc:
    resource_group: raoul_is_really_constrainst
    extends: .common-doc
    stage: test
    script:
    - sphinx-build -b html -d _build/doctrees docs _build/html -v
    - mkdir -p test
    - echo "CI_PIPELINE_ID=${CI_PIPELINE_ID}"
    - mv _build/html test/${CI_PIPELINE_ID}
    artifacts:
        name: "$CI_PIPELINE_ID"
        paths:
        - test
        expire_in: 5 days
    except:
    - master
    - develop
    - /^[0-9]+\.[0-9]+.*/

# Package
#
# Requires the following variables:
# - TWINE_PASSWORD
# - TWINE_USERNAME
# TODO: Reuse installation done in .common-dev
pypi9:
    extends: .common-dev
    variables:
      OS: ubuntu
      OS_VERSION: "22.04"
      OTB_VERSION: "${OTB9_VERSION}"
    stage: deploy
    script:
      # Upload only EGG as Wheel has formal dependency to gdal.
      - python3 -m pip install --upgrade twine
      - ls dist/
      - echo python3 -m twine upload --repository pypi dist/s1tiling-*.tar.gz
      # TODO: test the variables and the files exist!
      - python3 -m twine upload --repository pypi dist/s1tiling-*.tar.gz
    dependencies:
      - package9
    rules:
      # only for tags
      - if: $CI_COMMIT_TAG != null

# Upload latest documentation when develop/master branches are updated
pages:  # Job name need to be pages, in deploy stage!
  extends: .common-doc
  stage: deploy
  # resource_group: documentation_cache
  resource_group: raoul_is_really_constrainst
  variables:
    OS: ubuntu
    OS_VERSION: "22.04"
    OTB_VERSION: "${OTB9_VERSION}"
  script:
    - mkdir public
    # TODO: lock access to public
    - rclone copy s3_otb:s1-tiling/www public || echo "Oups| no www/ on S3"
    # Check cache
    - ls public
    - docs/install_and_deploy_doc.sh public
    - rclone copy public s3_otb:s1-tiling/www
  artifacts:
    paths:
      - public
  environment:
    name: generate_docs
  rules:
    # only for tags, master & develop
    - if: $CI_COMMIT_TAG != null
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_COMMIT_BRANCH == "develop"

#======================================================================
#
# Docker generation
#
.prepare-docker-config: &prepare-docker-config
  - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}"
           > /kaniko/.docker/config.json

.common-docker-build:
  image:
    name: gcr.io/kaniko-project/executor:v1.6.0-debug
    entrypoint: [""]
  variables:
    # Cannot work because of $(sh ...
    FULL_VERSION: $(sh ${CI_PROJECT_DIR}/docker/version_name.sh)-${OS}-otb${OTB_VERSION}
    BASE_FULL_VERSION: ${S1TD_VERSION}-${OS}${OS_VERSION}-otb${OTB_VERSION}
  before_script:
    - *prepare-docker-config
  script:
    - echo "CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME"
    - sh $CI_PROJECT_DIR/docker/version_name.sh
    - echo "> docker build ${CI_REGISTRY_IMAGE}:$(sh ${CI_PROJECT_DIR}/docker/version_name.sh)-${OS}-otb${OTB_VERSION} <-- ${BASE_FULL_VERSION}"
    - /kaniko/executor --cleanup
                       --verbosity info
                       --context "${CI_PROJECT_DIR}/"
                       --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile"
                       --destination "${CI_REGISTRY_IMAGE}:$(sh ${CI_PROJECT_DIR}/docker/version_name.sh)-${OS}-otb${OTB_VERSION}"
                       --build-arg BASE_FULL_VERSION
                       --build-arg CI_COMMIT_SHA
                       --build-arg CREATION_DATE="$(date -u '+%Y-%m-%dT%T%z')"
                       --build-arg S1TILING_VERSION=$(sh $CI_PROJECT_DIR/docker/version_name.sh)

#======================================================================
#
# Effective stages
#

test-doc9:
  extends: .test-doc
  variables:
    OS: ubuntu
    OS_VERSION: "22.04"
    OTB_VERSION: "${OTB9_VERSION}"

pytest9:
  extends: .pytest
  variables:
    OS: ubuntu
    OS_VERSION: "22.04"
    OTB_VERSION: "${OTB9_VERSION}"
    STEP: user

ubuntu-user9:
  extends: .common-docker-build
  stage: deploy
  variables:
    OS: ubuntu
    OS_VERSION: "22.04"
    OTB_VERSION: "${OTB9_VERSION}"
    STEP: user
  rules:
    # only for tags
    - if: $CI_COMMIT_TAG != null

# vim: set sw=2: