Multiple Version Images (#37)

luxonis · Oct 11, 2024 · 376efcc · 376efcc
1 parent 9f868fc
commit 376efcc
Show file tree

Hide file tree

Showing 37 changed files with 823 additions and 372 deletions.
diff --git a/.github/workflows/hailo_publish.yaml b/.github/workflows/hailo_publish.yaml
@@ -10,8 +10,15 @@ on:
 
 jobs:
   hailo-publish:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ["2024.04", "2024.07"]
+
     uses: ./.github/workflows/publish.yaml
     secrets: inherit
     with:
       package: hailo
+      version: ${{ matrix.version }}
 
diff --git a/.github/workflows/hailo_test.yaml b/.github/workflows/hailo_test.yaml
@@ -16,8 +16,15 @@ on:
 
 jobs:
   hailo-test:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ["2024.04", "2024.07"]
+
     uses: ./.github/workflows/modelconverter_test.yaml
     secrets: inherit
     with:
       package: hailo
+      version: ${{ matrix.version }}
 
diff --git a/.github/workflows/modelconverter_test.yaml b/.github/workflows/modelconverter_test.yaml
@@ -8,6 +8,12 @@ on:
         type: string
         description: Which package to test
 
+      version:
+        required: true
+        type: string
+        description: >
+          Which version of underlying conversion tools to use
+
 permissions:
   contents: read
   packages: read
@@ -17,6 +23,7 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_S3_ENDPOINT_URL: ${{ secrets.AWS_S3_ENDPOINT_URL }}
   PACKAGE: ${{ inputs.package }}
+  VERSION: ${{ inputs.version }}
 
 jobs:
   tests:
@@ -53,19 +60,24 @@ jobs:
         credentials_json: ${{ secrets.GCP_CREDENTIALS }}
         token_format: access_token
 
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v1
-
-    - name: Download file from GCS
+    - name: Download build files from GCS
+      if: ${{ inputs.package }} != hailo
+      env:
+        GS_BUILD_ARTIFACTS: gs://luxonis-test-bucket/modelconverter/build-artifacts
       run: |
+        mkdir -p docker/extra_packages
         cd docker/extra_packages
-        if [ "$PACKAGE" = "rvc4" ]; then
-            gsutil cp gs://luxonis-test-bucket/modelconverter/build-artifacts/snpe.zip .
-        elif [ "$PACKAGE" = "rvc2" ] || [ "$PACKAGE" = "rvc3" ]; then
-            gsutil cp gs://luxonis-test-bucket/modelconverter/build-artifacts/openvino_2022_3_vpux_drop_patched.tar.gz .
+        if [ "${PACKAGE}" = "rvc4" ]; then
+            gcloud storage cp                             \
+              "${GS_BUILD_ARTIFACTS}/snpe-${VERSION}.zip" \
+              "snpe-${VERSION}.zip"
+        elif [ "${PACKAGE}" = "rvc2" ] || [ "${PACKAGE}" = "rvc3" ]; then
+            gcloud storage cp                                    \
+              "${GS_BUILD_ARTIFACTS}/openvino-${VERSION}.tar.gz" \
+              "openvino-${VERSION}.tar.gz"
         fi
 
     - name: Run Tests
       run: |
-        pytest -s --verbose "tests/test_packages/test_$PACKAGE.py"
+        pytest -s --verbose "tests/test_packages/test_${PACKAGE}.py" --tool-version "${VERSION}"
 
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -8,21 +8,31 @@ on:
         type: string
         description: Which package to publish
 
+      version:
+        required: true
+        type: string
+        description: >
+          Which version of underlying conversion tools to use
+
 permissions:
   contents: read
   packages: write
 
 env:
-  PROJECT_ID: easyml-394818
   GAR_LOCATION: us-central1
   DOCKERFILE: docker/${{ inputs.package }}/Dockerfile
   PACKAGE: ${{ inputs.package }}
   NAME: luxonis/modelconverter-${{ inputs.package }}
   STEM: modelconverter-${{ inputs.package }}
+  VERSION: ${{ inputs.version }}
+  GS_BUILD_ARTIFACTS: gs://luxonis-test-bucket/modelconverter/build-artifacts
+  GAR_STEM: us-central1-docker.pkg.dev/easyml-394818
 
 jobs:
   ghcr-publish:
     runs-on: ubuntu-latest
+    env:
+      LOCAL_NAME: ${NAME}:${VERSION}-latest
 
     steps:
     - name: Checkout code
@@ -35,58 +45,6 @@ jobs:
         credentials_json: ${{ secrets.GCP_CREDENTIALS }}
         token_format: access_token
 
-    - name: Docker login to GHCR
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Get modelconverter version
-      id: commit
-      run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v1
-
-    - name: Download file from GCS
-      run: |
-        cd docker/extra_packages
-        if [ "$PACKAGE" = "rvc4" ]; then
-            gsutil cp gs://luxonis-test-bucket/modelconverter/build-artifacts/snpe.zip snpe.zip
-        elif [ "$PACKAGE" = "rvc2" ] || [ "$PACKAGE" = "rvc3" ]; then
-            gsutil cp gs://luxonis-test-bucket/modelconverter/build-artifacts/openvino_2022_3_vpux_drop_patched.tar.gz openvino_2022_3_vpux_drop_patched.tar.gz
-        fi
-    - name: Publish latest
-      run: |
-        docker build -f $DOCKERFILE -t $NAME:latest .
-        docker tag $NAME:latest ghcr.io/$NAME:latest
-        docker push ghcr.io/$NAME:latest
-
-    - name: Publish tagged
-      run: |
-        VERSION=${{ steps.commit.outputs.sha }}
-        docker tag $NAME:latest ghcr.io/$NAME:$VERSION
-        docker push ghcr.io/$NAME:$VERSION
-
-  gar-publish:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Get modelconverter version
-      id: commit
-      run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-
-    - name: Authenticate to Google Cloud
-      id: google-auth
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.GCP_CREDENTIALS }}
-        token_format: access_token
-
     - name: Docker login to GAR
       uses: docker/login-action@v3
       with:
@@ -104,30 +62,56 @@ jobs:
     - name: Set up Cloud SDK
       uses: google-github-actions/setup-gcloud@v1
 
-    - name: Download file from GCS
+    - name: Download build files from GCS
+      if: ${{ inputs.package }} != hailo
       run: |
         cd docker/extra_packages
         if [ "$PACKAGE" = "rvc4" ]; then
-            gsutil cp gs://luxonis-test-bucket/modelconverter/build-artifacts/snpe.zip .
-        elif [ "$PACKAGE" = "rvc2" ] || [ "$PACKAGE" = "rvc3" ]; then
-            gsutil cp gs://luxonis-test-bucket/modelconverter/build-artifacts/openvino_2022_3_vpux_drop_patched.tar.gz .
+            gcloud storage cp                             \
+              "${GS_BUILD_ARTIFACTS}/snpe-${VERSION}.zip" \
+              "snpe-${VERSION}.zip"
+        elif [ "${PACKAGE}" = "rvc2" ] || [ "${PACKAGE}" = "rvc3" ]; then
+            gcloud storage cp                                    \
+              "${GS_BUILD_ARTIFACTS}/openvino-${VERSION}.tar.gz" \
+              "openvino-${VERSION}.tar.gz"
         fi
 
-    - name: Publish
+    - name: Build image
+      run: |
+        docker build -f "${DOCKERFILE}" -t "${LOCAL_NAME}" .
+
+    - name: GHCR publish latest
       run: |
-        docker build -f $DOCKERFILE -t $NAME:latest .
+        GHCR_NAME="ghcr.io/${LOCAL_NAME}"
+        docker tag "${LOCAL_NAME}" "${GHCR_NAME}"
+        docker push "${GHCR_NAME}"
 
-        VERSION=${{ steps.commit.outputs.sha }}
-        GAR_NAME="$GAR_LOCATION-docker.pkg.dev/$PROJECT_ID/internal/$STEM"
+    - name: GHCR publish SHA
+      run: |
+        SHA=$(git rev-parse --short HEAD)
+        GHCR_NAME="ghcr.io/${NAME}:${VERSION}-${SHA}"
+        docker tag "${LOCAL_NAME}" "${GHCR_NAME}"
+        docker push "${GHCR_NAME}"
 
-        for tag in latest $VERSION; do
-            docker tag "$NAME:latest" "$GAR_NAME:$tag"
-            docker push "$GAR_NAME:$tag"
-        done
+    - name: GAR publish latest
+      run: |
+        GAR_NAME="${GAR_STEM}/internal/${STEM}:${VERSION}-latest"
+        docker tag "${LOCAL_NAME}" "${GAR_NAME}"
+        docker push "${GAR_NAME}"
+
+    - name: GAR publish SHA
+      run: |
+        SHA=$(git rev-parse --short HEAD)
+        GAR_NAME="${GAR_STEM}/internal/${STEM}:${VERSION}-${SHA}"
+        docker tag "${LOCAL_NAME}" "${GAR_NAME}"
+        docker push "${GAR_NAME}"
+
+    - name: GAR publish clients
+      run: |
+        read -r -a REPO_ARRAY <<< "${{ vars.EXTERNAL_CLIENTS }}"
 
-        IFS=' ' read -r -a REPO_ARRAY <<< "${{ vars.EXTERNAL_CLIENTS }}"
         for REPO in "${REPO_ARRAY[@]}"; do
-          GAR_CLIENT_TAG="$GAR_LOCATION-docker.pkg.dev/$PROJECT_ID/$REPO/$STEM:$VERSION"
-            docker tag "$NAME:latest" "$GAR_CLIENT_TAG"
-            docker push "$GAR_CLIENT_TAG"
+          GAR_CLIENT_NAME="${GAR_STEM}/${REPO}/${STEM}:${VERSION}-${SHA}"
+          docker tag "${LOCAL_NAME}" "${GAR_CLIENT_NAME}"
+          docker push "${GAR_CLIENT_NAME}"
         done
diff --git a/.github/workflows/rvc2_publish.yaml b/.github/workflows/rvc2_publish.yaml
@@ -10,7 +10,14 @@ on:
 
 jobs:
   rvc2-publish:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ["2021.4.0", "2022.3.0"]
+
     uses: ./.github/workflows/publish.yaml
     secrets: inherit
     with:
       package: rvc2
+      verison: ${{ matrix.version }}
diff --git a/.github/workflows/rvc2_test.yaml b/.github/workflows/rvc2_test.yaml
@@ -16,7 +16,14 @@ on:
 
 jobs:
   rvc2-test:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ["2021.4.0", "2022.3.0"]
+
     uses: ./.github/workflows/modelconverter_test.yaml
     secrets: inherit
     with:
       package: rvc2
+      version: ${{ matrix.version }}
diff --git a/.github/workflows/rvc3_publish.yaml b/.github/workflows/rvc3_publish.yaml
@@ -14,3 +14,4 @@ jobs:
     secrets: inherit
     with:
       package: rvc3
+      version: "2022.3.0"
diff --git a/.github/workflows/rvc3_test.yaml b/.github/workflows/rvc3_test.yaml
@@ -20,3 +20,4 @@ jobs:
     secrets: inherit
     with:
       package: rvc3
+      version: "2022.3.0"
diff --git a/.github/workflows/rvc4_publish.yaml b/.github/workflows/rvc4_publish.yaml
@@ -10,7 +10,19 @@ on:
 
 jobs:
   rvc4-publish:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "2.23.0"
+          - "2.24.0"
+          - "2.25.0"
+          - "2.26.2"
+          - "2.27.0"
+
     uses: ./.github/workflows/publish.yaml
     secrets: inherit
     with:
       package: rvc4
+      version: ${{ matrix.version }}
diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml
@@ -16,7 +16,19 @@ on:
 
 jobs:
   rvc4-test:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "2.23.0"
+          - "2.24.0"
+          - "2.25.0"
+          - "2.26.2"
+          - "2.27.0"
+
     uses: ./.github/workflows/modelconverter_test.yaml
     secrets: inherit
     with:
       package: rvc4
+      version: ${{ matrix.version }}
diff --git a/README.md b/README.md
@@ -51,20 +51,33 @@ Otherwise follow the installation instructions for your OS from the [official we
 
 `ModelConverter` is in an experimental public beta stage. Some parts might change in the future.
 
-To build the images, you need to download additional packages depending on the selected target.
+To build the images, you need to download additional packages depending on the selected target and the desired version of the underlying conversion tools.
 
-**RVC2 and RVC3**
+**RVC2**
 
-Requires `openvino_2022_3_vpux_drop_patched.tar.gz` to be present in `docker/extra_packages`.
-You can download the archive [here](https://drive.google.com/file/d/1IXtYi1Mwpsg3pr5cDXlEHdSUZlwJRTVP/view?usp=share_link).
+Requires `openvino-<version>.tar.gz` to be present in `docker/extra_packages/`.
+
+- Version `2023.2.0` archive can be downloaded from [here](https://drive.google.com/file/d/1IXtYi1Mwpsg3pr5cDXlEHdSUZlwJRTVP/view?usp=share_link).
+
+- Version `2021.4.0` archive can be downloaded from [here](https://storage.openvinotoolkit.org/repositories/openvino/packages/2021.4/l_openvino_toolkit_dev_ubuntu20_p_2021.4.582.tgz)
+
+You only need to rename the archive to either `openvino-2023.2.0.tar.gz` or `openvino-2021.4.0.tar.gz` and place it in the `docker/extra_packages` directory.
+
+**RVC3**
+
+Only the version `2023.2.0` of `OpenVino` is supported for `RVC3`. Follow the instructions for `RVC2` to use the correct archive.
 
 **RVC4**
 
-Requires `snpe.zip` archive to be present in `docker/extra_packages`. You can download an archive with the current version [here](https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.23.0.24.06.24.zip). You only need to rename it to `snpe.zip` and place it in the `docker/extra_packages` directory.
+Requires `snpe-<version>.zip` archive to be present in `docker/extra_packages`. You can download version `2.23.0` from [here](https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.23.0.24.06.24.zip). You only need to rename it to `snpe-2.23.0.zip` and place it in the `docker/extra_packages` directory.
 
 **HAILO**
 
-Requires `hailo_ai_sw_suite_2024-04:1` docker image to be present on the system. You can download the image from the [Hailo website](https://developer.hailo.ai/developer-zone/sw-downloads/). Furthermore, you need to use the `docker/hailo/Dockerfile.public` file to build the image. The `docker/hailo/Dockerfile` is for internal use only.
+Requires `hailo_ai_sw_suite_<version>:1` docker image to be present on the system. You can obtain the image by following the instructions on [Hailo website](https://developer.hailo.ai/developer-zone/sw-downloads/).
+
+After you obtain the image, you need to rename it to `hailo_ai_sw_suite_<version>:1` using `docker tag <old_name> hailo_ai_sw_suite_<version>:1`.
+
+Furthermore, you need to use the `docker/hailo/Dockerfile.public` file to build the image. The `docker/hailo/Dockerfile` is for internal use only.
 
 ### Instructions