.gitlab-ci.yml

stages:
  - pylint_test
  - dev_test
  - pip_test
  - pip_publish
  - release_notes

pylintjob:
  stage: pylint_test
  except:
    - 139-correct-toynet-demo-instructions
    - 148-publish-niftynet-v0-2-0-on-python-package-index-pypi
    - 170-add-niftynet-paper-on-rtd-doc
    - 167-document-cli-option-for-path-to-new-networks
    - 176-document-pip-installer-bundling-guidelines
    - 174-design-a-workflow-that-allows-prs-from-github-to-be-merged
    - 206-improve-the-handling-of-release-notes
    - 221-add-changelog-entry-for-version-0.2.2
    - 223-put-bug-fixes-under-fixed-header-in-changelog
  script:
    - pylint --rcfile=tests/pylintrc niftynet/engine
    - pylint --rcfile=tests/pylintrc niftynet/io
    - pylint --rcfile=tests/pylintrc niftynet/utilities/user_parameters_*py
    - pylint --rcfile=tests/pylintrc niftynet/utilities/download.py
    - pylint --rcfile=tests/pylintrc niftynet/utilities/niftynet_global_config.py
  tags:
    - gift-little

testjob:
  stage: dev_test
  only:
    - master
    - dev
    - tags
  script:
    # !!kill coverage in case of hanging processes
    - if pgrep coverage; then pkill -f coverage; fi

    - export TF_CPP_MIN_LOG_LEVEL=2
    # print system info
    - which nvidia-smi
    - nvidia-smi
    - pwd
    - ls -la /dev | grep nvidia

    # removing existing testing data
    - if [ -d models ]; then rm -r models; fi
    - if [ -d testing_data ]; then rm -r testing_data; fi
    - if [ -d example_volumes ]; then rm -r example_volumes; fi

    # download data
    - wget -q https://www.dropbox.com/s/lioecnpv82r5n6e/example_volumes_v0_2.tar.gz
    - tar -xzvf example_volumes_v0_2.tar.gz
    - wget -q https://www.dropbox.com/s/2g8jr3wq8rw5xtv/testing_code_v0_3.tar.gz
    - wget -q https://www.dropbox.com/s/5p5fdgy053tgmdj/testing_data_v0_3.tar.gz
    - mkdir -p testing_data
    - tar -xzvf testing_data_v0_3.tar.gz -C testing_data
    - tar -xzvf testing_code_v0_3.tar.gz -C testing_data
    - wget -N https://www.dropbox.com/s/gt0hm6o61rlsfcc/csv_data.tar.gz
    - tar -C data -xzvf csv_data.tar.gz

    ######### Python 3 ###################### run python3 code with coverage wrapper
    # save NiftyNet folder path just in case
    - export niftynet_dir=$(pwd)

    # create a virtual env to dev-test
    - venv="niftynet-dev-test-py3"
    - mypython=$(which python3)
    - virtualenv -p $mypython $venv
    - cd $venv
    - venv_dir=$(pwd)
    - source bin/activate
    # print Python version to CI output
    - which python
    - python --version
    - cd $niftynet_dir

    - pip install coverage
    - pip install --upgrade pip
    - pip install -r requirements.txt

    - echo $(python tests/get_gpu_index.py)
    - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)

    - python net_download.py testing -r
    - coverage run -a --source . net_segment.py train -c config/highres3dnet_config.ini --batch_size=1 --num_threads=2 --queue_length=40 --max_iter=10
    - coverage run -a --source . net_segment.py inference -c config/highres3dnet_config.ini --batch_size 8 --spatial_window_size 64,64,64 --queue_length 64

    - coverage run -a --source . net_segment.py train -c config/scalenet_config.ini --batch_size 1 --queue_length 5 --num_threads 2
    - coverage run -a --source . net_segment.py inference -c config/scalenet_config.ini --batch_size 16 --spatial_window_size 64,64,64 --queue_length 32

    - coverage run -a --source . net_segment.py train -c config/vnet_config.ini --batch_size 1 --queue_length 5 --num_threads 2 --activation_function relu
    - coverage run -a --source . net_segment.py inference -c config/vnet_config.ini --batch_size 16 --spatial_window_size 64,64,64 --queue_length 32 --activation_function relu

    # need a large GPU to run
    - coverage run -a --source . net_segment.py train -c config/unet_config.ini --batch_size 1 --queue_length 5 --num_threads 2
    - coverage run -a --source . net_segment.py inference -c config/unet_config.ini --batch_size 1 --spatial_window_size 96,96,96 --queue_length 5

      #- coverage run -a --source . net_segment.py train -c config/deepmedic_config.ini --batch_size 128 --queue_length 48 --num_threads 4
      #- coverage run -a --source . net_segment.py inference -c config/deepmedic_config.ini --batch_size 12 --spatial_window_size 135,135,135  --queue_length 128

    - coverage run -a --source . net_segment.py train -c config/default_segmentation.ini --batch_size 3 --queue_length 6
    - coverage run -a --source . net_segment.py train -c config/default_segmentation.ini --batch_size 3 --queue_length 6 --exclude_fraction_for_validation 0.1 --validation_every_n 3
    - coverage run -a --source . net_segment.py train -c config/default_segmentation.ini --batch_size 3 --queue_length 6 --starting_iter 0 --max_iter 15
    - coverage run -a --source . net_segment.py inference -c config/default_segmentation.ini --spatial_window_size 84,84,84 --batch_size 7 --queue_length 14 --inference_iter 15
    - coverage run -a --source . net_segment.py evaluation -c config/default_segmentation.ini

    - coverage run -a --source . net_segment.py train -c config/default_multimodal_segmentation.ini --batch_size 3
    - coverage run -a --source . net_segment.py inference -c config/default_multimodal_segmentation.ini --spatial_window_size 64,64,64 --batch_size 7

    - coverage run -a --source .  net_classify.py train -c testing_data/test_classification.ini
    - coverage run -a --source .  net_classify.py inference -c testing_data/test_classification.ini
      # - coverage run -a --source .  net_classify.py evaluation -c testing_data/test_classification.ini

    - coverage run -a --source . net_regress.py train -c config/default_monomodal_regression.ini --max_iter 10 --name toynet --batch_size=2
    - coverage run -a --source . net_run.py train -a net_regress -c config/default_monomodal_regression.ini --max_iter 10 --name toynet --batch_size=2
    - coverage run -a --source . net_regress.py inference -c config/default_monomodal_regression.ini --name toynet --spatial_window_size 84,84,84 --batch_size 7
    - coverage run -a --source . net_regress.py evaluation -c config/default_monomodal_regression.ini

    - coverage run -a --source . net_gan.py train -c config/GAN_demo_train_config.ini --max_iter 5
    - coverage run -a --source . net_run.py train -a net_gan -c config/GAN_demo_train_config.ini --max_iter 5
    - coverage run -a --source . net_gan.py inference -c config/GAN_demo_train_config.ini

    - coverage run -a --source . net_autoencoder.py train -c config/vae_config.ini --max_iter 5
    - coverage run -a --source . net_autoencoder.py inference -c config/vae_config.ini --inference_type sample
    - coverage run -a --source . net_autoencoder.py inference -c config/vae_config.ini --inference_type encode --save_seg_dir output/vae_demo_features
    - coverage run -a --source . net_autoencoder.py inference -c config/vae_config.ini --inference_type encode-decode

    - coverage run -a --source . net_download.py dense_vnet_abdominal_ct_model_zoo -r

    - coverage run -a --source . -m tests.test_model_zoo
    - coverage run -a --source . -m unittest discover -s "tests" -p "*_test.py"
    - coverage report --omit='niftynet-dev-test*' -m

    # deactivate virtual environment
    - deactivate
    - cd $niftynet_dir
    ###############end of python3
    - echo 'finished test'
  tags:
    - gift-little

quicktest:
  stage: dev_test
  except:
    - master
    - dev
    - tags
    - 147-revise-contribution-guidelines-to-include-github
    - 150-properly-format-the-bibtex-entry-to-the-ipmi-2017-paper-on-the-main-readme
    - 148-publish-niftynet-v0-2-0-on-python-package-index-pypi
    - 170-add-niftynet-paper-on-rtd-doc
    - 167-document-cli-option-for-path-to-new-networks
    - 176-document-pip-installer-bundling-guidelines
    - 174-design-a-workflow-that-allows-prs-from-github-to-be-merged
    - 206-improve-the-handling-of-release-notes
    - 221-add-changelog-entry-for-version-0.2.2
    - 223-put-bug-fixes-under-fixed-header-in-changelog
    - 195-add-evaluation-action-3
  script:
    # print system info
    - which nvidia-smi
    - nvidia-smi
    - pwd
    - ls -la /dev | grep nvidia

    - echo $(python tests/get_gpu_index.py)
    - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)

    # removing existing testing data
    - if [ -d models ]; then rm -r models; fi
    - if [ -d testing_data ]; then rm -r testing_data; fi
    - if [ -d example_volumes ]; then rm -r example_volumes; fi

    # download data
    - wget -q https://www.dropbox.com/s/lioecnpv82r5n6e/example_volumes_v0_2.tar.gz
    - tar -xzvf example_volumes_v0_2.tar.gz
    - wget -q https://www.dropbox.com/s/5p5fdgy053tgmdj/testing_data_v0_3.tar.gz
    - mkdir -p testing_data
    - tar -xzvf testing_data_v0_3.tar.gz -C testing_data
    - wget -q https://www.dropbox.com/s/gt0hm6o61rlsfcc/csv_data.tar.gz
    - tar -C data -xzvf csv_data.tar.gz

    ######### Python 3 ###################### run python3 code with coverage wrapper
    # save NiftyNet folder path just in case
    - export niftynet_dir=$(pwd)

    # create a virtual env to dev-test
    - venv="niftynet-dev-test-py3"
    - mypython=$(which python3)
    - virtualenv -p $mypython $venv
    - cd $venv
    - venv_dir=$(pwd)
    - source bin/activate
    # print Python version to CI output
    - which python
    - python --version
    - cd $niftynet_dir

    - pip install coverage
    - pip install --upgrade pip
    - pip install -r requirements.txt

    - python -c "import tensorflow as tf; print(tf.__version__)"
    #- python -c "import tensorflow as tf; from tensorflow.python.client import device_lib; print device_lib.list_local_devices()"
    - coverage erase
    # run only fast tests
    - echo $(python tests/get_gpu_index.py)
    - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)

    - python net_download.py testing -r
    - QUICKTEST=True coverage run -a --source . -m unittest discover -s "tests" -p "*_test.py"
    - coverage report --omit='niftynet-dev-test*' -m
    # deactivate virtual environment
    - deactivate
    - cd $niftynet_dir
    ###############end of python3

    - echo 'finished quick tests'
  tags:
    - gift-little


pip-installer:
  stage: pip_test
  only:
    - master
    - dev
    - tags
  script:
    # get the shortened version of last commit's hash
    - LAST_COMMIT=$(git rev-parse --short HEAD)
    # following three lines copied over from dev script:
    - ls -la /dev | grep nvidia
    - export niftynet_dir=$(pwd)
    # ============= Python 3 ============================
    # create a virtual env to test pip installer
    - venv="niftynet-pip-installer-venv-py3"
    - mypython=$(which python3)
    - virtualenv -p $mypython $venv
    - cd $venv
    - venv_dir=$(pwd)
    - source bin/activate
    # print Python version to CI output
    - which python
    - python --version
    # NiftyNet console entries should fail gracefully if TF not installed
    # i.e. check that the warning displays the TF website
    - cd $niftynet_dir
    - echo $(python tests/get_gpu_index.py)
    - export CUDA_VISIBLE_DEVICES=$(python tests/get_gpu_index.py)

    # create a Python file that will import all available packages from the pip installer
    - package_importer="$(pwd)/import_niftynet_packages.py"
    # traverse the file hierarchy recursively to discover all packages
    - find niftynet -type f \( ! -name . \) ! -path "niftynet/contrib/niftyreg_image_resampling*" -print | grep '.py$' | grep -v __init__ | sed 's/\.\.\///g;s/\//\./g;s/\.py//g;s/^niftynet/import niftynet/g' > $package_importer

    - set +e
    - python -c "import niftynet" 2>&1 | grep "https://www.tensorflow.org/"
    - set -e
    # install TF
    - pip install --upgrade pip
    - pip install "tensorflow==1.15.*"
    - pip install -r requirements.txt


    # source utils
    - source ci/utils.sh
    # create the NiftyNet wheel
    - rm -rf dist  # remove dist directory, just in case
    - sh ci/bundlewheel.sh
    - source ci/findwheel.sh
    - echo $niftynet_wheel
    - pip install $niftynet_wheel
    # install SimpleITK for package importer test to work properly
    - pip install simpleitk
    # check whether all packages are importable
    - cat $package_importer
    - python $package_importer
    # test niftynet command
    - ln -s /home/gitlab-runner/environments/niftynet/data/example_volumes ./example_volumes
    - net_download testing -r

    - net_segment train -c $niftynet_dir/config/default_segmentation.ini --name toynet --batch_size 3 --max_iter 5
    - net_segment inference -c $niftynet_dir/config/default_segmentation.ini --name toynet --spatial_window_size 80,80,80 --batch_size 8
    - net_segment evaluation -c $niftynet_dir/config/default_segmentation.ini
    - net_run train --app net_segment -c $niftynet_dir/config/default_segmentation.ini --name toynet --batch_size 3 --max_iter 5
    - net_run inference --app net_segment -c $niftynet_dir/config/default_segmentation.ini --name toynet --spatial_window_size 80,80,80 --batch_size 8
    - net_run evaluation --app net_segment -c $niftynet_dir/config/default_segmentation.ini

    - net_classify train -c extensions/testing/test_classification.ini
    - net_classify inference -c extensions/testing/test_classification.ini
      #- net_classify evaluation -c extensions/testing/test_classification.ini
    - net_run --app net_classify train -c extensions/testing/test_classification.ini
    - net_run --app net_classify inference -c extensions/testing/test_classification.ini
      #- net_run --app net_classify evaluation -c extensions/testing/test_classification.ini

    - net_regress train -c $niftynet_dir/config/default_monomodal_regression.ini --max_iter 10 --name toynet --batch_size=2
    - net_regress inference -c $niftynet_dir/config/default_monomodal_regression.ini --name toynet --spatial_window_size 84,84,84 --batch_size 7
    - net_regress evaluation -c $niftynet_dir/config/default_monomodal_regression.ini
    - net_run train -a net_regress -c $niftynet_dir/config/default_monomodal_regression.ini --max_iter 10 --name toynet --batch_size=2
    - net_run inference -a net_regress -c $niftynet_dir/config/default_monomodal_regression.ini --name toynet --spatial_window_size 84,84,84 --batch_size 7
    - net_run evaluation -a net_regress -c $niftynet_dir/config/default_monomodal_regression.ini

    - net_gan train -c $niftynet_dir/config/GAN_demo_train_config.ini --max_iter 5
    - net_gan inference -c $niftynet_dir/config/GAN_demo_train_config.ini
    - net_run train --app net_gan -c $niftynet_dir/config/GAN_demo_train_config.ini --max_iter 5
    - net_run inference --app net_gan -c $niftynet_dir/config/GAN_demo_train_config.ini

    - net_autoencoder train -c $niftynet_dir/config/vae_config.ini --max_iter 5
    - net_autoencoder inference -c $niftynet_dir/config/vae_config.ini --inference_type sample
    - net_autoencoder inference -c $niftynet_dir/config/vae_config.ini --inference_type encode --save_seg_dir output/vae_demo_features
    - net_autoencoder inference -c $niftynet_dir/config/vae_config.ini --inference_type encode-decode
    - net_run train --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --max_iter 5
    - net_run inference --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --inference_type sample
    - net_run inference --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --inference_type encode --save_seg_dir output/vae_demo_features
    - net_run inference --app net_autoencoder -c $niftynet_dir/config/vae_config.ini --inference_type encode-decode

    - net_download dense_vnet_abdominal_ct_model_zoo -r
    # deactivate virtual environment
    - deactivate
    - cd $niftynet_dir
  tags:
    - gift-little

pip-camera-ready:
  stage: pip_publish
  only:
    - tags
  script:
    - export check_version_venv=check_version_venv
    - virtualenv -p $(which python3) $check_version_venv
    - source $check_version_venv/bin/activate
    - pip install "tensorflow==1.15.*"
    - pip install -r requirements.txt
    # Copy wheel created in previous stage to a specific location on GIFT-Adelie
    - export niftynet_dir=$(pwd)
    # create the NiftyNet wheel
    - rm -rf dist  # remove dist directory, just in case
    - source ci/utils.sh
    - sh ci/bundlewheel.sh
    - source ci/findwheel.sh
    - echo $niftynet_wheel
    # Creat camera-ready folder if doesn't exist
    - camera_ready_dir=/home/gitlab-runner/environments/niftynet/pip/camera-ready
    - mkdir -p $camera_ready_dir
    - ls -lrtha $camera_ready_dir
    # Clean up the camera-ready folder if already there
    - rm -rf $camera_ready_dir/*.whl
    # check whether version command works
    - pip3 install "tensorflow==1.15.*"
    - pip3 install -r requirements.txt
    - pip3 install $niftynet_wheel
    - net_segment --version
    - net_segment --version 2>&1 | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+' | grep -v +
    - python -c 'import niftynet; print(niftynet.__version__)' | grep -E 'NiftyNet.*version.*[0-9]+\.[0-9]+\.[0-9]+' | grep -v +
    - deactivate
    - rm -rf $check_version_venv
    # Finally do copy
    - cp $niftynet_wheel $camera_ready_dir
    - ls -lrtha $camera_ready_dir
    # Instruct developer which file to publish
    - echo "Camera-ready pip installer bundle (wheel) created:"
    - echo "$(ls $camera_ready_dir/*.whl)"
  tags:
    - gift-little

release-notes:
  stage: release_notes
  only:
    - tags
  script:
    - echo $CI_COMMIT_REF_NAME
    - echo $CI_COMMIT_TAG
    # Fail if nothing's been added to changelog for this version
    - echo "Checking changelog modified for current release ..."
    - grep ${CI_COMMIT_TAG:1} CHANGELOG.md
    - echo "... DONE"
    # Do actually push release notes to GitHub
    - |
        if [[ "$CI_COMMIT_TAG" == v* ]]; then
          echo "Doing a chandler push for $CI_COMMIT_TAG ..."
          chandler push $CI_COMMIT_TAG --github=NifTK/NiftyNet
          echo "... DONE"
        fi
  tags:
    - gift-little