diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index f577603b..7b4d5e24 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -11,106 +11,151 @@ on: branches: - main - staging-* +env: + http_proxy: ${{ secrets.PROXY_URL }} + https_proxy: ${{ secrets.PROXY_URL }} jobs: - CICD: + # CICD: + # runs-on: self-hosted + + # steps: + # - name: Checkout branch + # uses: actions/checkout@v4 + + # - name: Build docker image + # run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d . + + # - name: Run pytest + # run: > + # docker run + # --ipc=host + # myria3d + # python -m + # pytest -rA -v + # --ignore=actions-runner + + # # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable + # # sufficient shared memory when predicting on large files. + # - name: Example inference run via Docker with default config and checkpoint + # run: > + # docker run + # -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ + # -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + # --ipc=host + # --shm-size=2gb + # myria3d + # python run.py + # predict.src_las=/inputs/792000_6272000_subset_buildings.las + # datamodule.epsg=2154 + # predict.output_dir=/outputs/ + # task.task_name=predict + + # # predict.subtile_overlap specifies overlap between adjacent samples (in meters). + # - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. + # run: > + # docker run + # -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ + # -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + # --ipc=host + # --shm-size=2gb + # myria3d + # python run.py + # --config-path /inputs/ + # --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 + # predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt + # datamodule.epsg=2154 + # predict.src_las=/inputs/792000_6272000_subset_buildings.las + # predict.output_dir=/outputs/ + # predict.subtile_overlap=25 + # datamodule.batch_size=10 + # predict.interpolator.probas_to_save=[building,ground] + # task.task_name=predict + + # - name: Check code neatness (linter) + # run: docker run myria3d python -m flake8 + + # # Everything ran so we tag the valid docker image to keep it + # # This happens for push events, which are in particular + # # triggered when a pull request is merged. + # - name: Tag the docker image with branch name + # if: github.event_name == 'push' + # run: | + # docker tag myria3d:latest myria3d:${{github.ref_name}} + # docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune + # # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar + + # # get version number and date, to tag the image pushed to a private docker registry + # - name: get version number + # id: tag + # run: | + # echo "VERSION=$(docker run myria3d python -m myria3d._version)" >> $GITHUB_ENV + # echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV + + # # show possible tags, for debugging purpose + # - name: Print tags + # run: | + # echo "${{ env.VERSION }}" + # echo "${{ env.DATE }}" + + # - name: push main docker on nexus (tagged with a date) + # # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) + # if: ((github.ref_name == 'main') && (github.event_name == 'push')) + # run: | + # docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} + # docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} + # docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} + + # - name: push branch docker on nexus (tagged with the branch name) + # # we push on nexus an image from a branch when it's pushed + # if: ((github.event_name == 'push') && (github.ref_name != 'main')) + # run: | + # docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} + # docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} + # docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} + + # - name: Clean dangling docker images + # if: always() # always do it, even if something failed + # run: docker system prune --force # remove dangling docker images, without asking user for confirmation + + publish-pypi: runs-on: self-hosted + environment: + name: pypi + url: https://pypi.org/p/myria3d/ + permissions: + contents: read + packages: write + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: + - name: Checkout branch + uses: actions/checkout@v4 + + # See https://github.com/marketplace/actions/setup-micromamba + - name: setup-micromamba + uses: mamba-org/setup-micromamba@v1.4.3 + with: + environment-file: environment.yml + environment-name: myria3d # activate the environment + cache-environment: true + cache-downloads: true + generate-run-shell: true + + - name: Run tests with pytest + shell: micromamba-shell {0} + run: python -m pytest ./test -s --log-cli-level DEBUG + + - name: Build pip package + shell: micromamba-shell {0} + run: > + rm -rf tmp + rm -rf ign_pdal_tools.egg-info + rm -rf dist + python -m build - - name: Checkout branch - uses: actions/checkout@v4 - - - name: Build docker image - run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d . - - - name: Run pytest - run: > - docker run - --ipc=host - myria3d - python -m - pytest -rA -v - --ignore=actions-runner - - # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable - # sufficient shared memory when predicting on large files. - - name: Example inference run via Docker with default config and checkpoint - run: > - docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ - --ipc=host - --shm-size=2gb - myria3d - python run.py - predict.src_las=/inputs/792000_6272000_subset_buildings.las - datamodule.epsg=2154 - predict.output_dir=/outputs/ - task.task_name=predict - - # predict.subtile_overlap specifies overlap between adjacent samples (in meters). - - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. - run: > - docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ - --ipc=host - --shm-size=2gb - myria3d - python run.py - --config-path /inputs/ - --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 - predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt - datamodule.epsg=2154 - predict.src_las=/inputs/792000_6272000_subset_buildings.las - predict.output_dir=/outputs/ - predict.subtile_overlap=25 - datamodule.batch_size=10 - predict.interpolator.probas_to_save=[building,ground] - task.task_name=predict - - - name: Check code neatness (linter) - run: docker run myria3d python -m flake8 - - # Everything ran so we tag the valid docker image to keep it - # This happens for push events, which are in particular - # triggered when a pull request is merged. - - name: Tag the docker image with branch name - if: github.event_name == 'push' - run: | - docker tag myria3d:latest myria3d:${{github.ref_name}} - docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune - # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar - - # get version number and date, to tag the image pushed to a private docker registry - - name: get version number - id: tag - run: | - echo "VERSION=$(docker run lidar_prod python -m myria3d._version)" >> $GITHUB_ENV - echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV - # show possible tags, for debugging purpose - - name: Print tags - run: | - echo "${{ env.VERSION }}" - echo "${{ env.DATE }}" - - - name: push main docker on nexus (tagged with a date) - # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) - if: ((github.ref_name == 'main') && (github.event_name == 'push')) - run: | - docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} - docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} - docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} - - - name: push branch docker on nexus (tagged with the branch name) - # we push on nexus an image from a branch when it's pushed - if: ((github.event_name == 'push') && (github.ref_name != 'main')) - run: | - docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} - docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} - docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} - - - name: Clean dangling docker images - if: always() # always do it, even if something failed - run: docker system prune --force # remove dangling docker images, without asking user for confirmation + - name: pypi-publish + if: ((github.ref_name == 'main') && (github.event_name == 'push')) + uses: pypa/gh-action-pypi-publish@v1.8.14 + with: + skip-existing: true diff --git a/docs/source/tutorials/install_on_linux.md b/docs/source/tutorials/install_on_linux.md index 2b9a66f9..50533e44 100644 --- a/docs/source/tutorials/install_on_linux.md +++ b/docs/source/tutorials/install_on_linux.md @@ -4,7 +4,7 @@ ### Prerequisites -We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments. +We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments. This makes installing pytorch-related libraries way easier than using pure pip installs. We enable CUDA-acceleration in pytorch as part of the defaut virtual environment recipe (see below). @@ -43,9 +43,17 @@ Start by activating the virtual environment with ```bash conda activate myria3d ``` -Then install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release. + +Then install the latest version from pypi. +**Warning:** activating the environment is required as the public pip package does not +handle its dependencies! ``` -pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name} +pip install myria3d +``` + +Or install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release. +``` +pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name} ``` Alternatively, you can install from sources directly in editable mode with @@ -53,10 +61,10 @@ Alternatively, you can install from sources directly in editable mode with pip install --editable . ``` - + ## Troubleshooting - *OSError(libcusparse.so.11 cannot open shared object file no such file or directory)* ([**](https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625)) - open the .bashrc file from your Ubuntu home directory and at the end of the file, add the following line (replace anaconda3 with miniconda3 if needed) - export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" diff --git a/environment.yml b/environment.yml index 17f3bf24..47f589c4 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - matplotlib # --------- loggers --------- # - comet_ml::comet_ml=3.35 - - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 + - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 # --------- Visualization --------- # - pandas - matplotlib @@ -57,17 +57,18 @@ dependencies: - rich # beautiful text formatting in terminal - sh # for running bash commands in some tests - pudb # debugger + - twine # to publish pip package # # --------- Documentation --------- # - sphinx==4.5.* - recommonmark==0.7.* - sphinx_rtd_theme==1.0.* - docutils==0.17 - - rstcheck==3.3.* # RST Linter + - rstcheck==3.3.* # RST Linter - pip: - - hydra-core==1.1.* - - hydra-colorlog==1.1.* - # --------- Documentation --------- # - - myst_parser==0.17.* - - sphinxnotes-mock==1.0.0b0 # still a beta - - sphinx_paramlinks==0.5.* - - ign-pdal-tools>=1.5.2 \ No newline at end of file + - hydra-core==1.1.* + - hydra-colorlog==1.1.* + # --------- Documentation --------- # + - myst_parser==0.17.* + - sphinxnotes-mock==1.0.0b0 # still a beta + - sphinx_paramlinks==0.5.* + - ign-pdal-tools>=1.5.2