feat: use ansible to download ONNX files (#3375)

* feat: use ansible to download ONNX files Signed-off-by: Esteve Fernandez <[email protected]> * feat: add remaining ONNX files Signed-off-by: Esteve Fernandez <[email protected]> * feat: add data_dir option to choose the location to store models Signed-off-by: Esteve Fernandez <[email protected]> * feat: update artifacts per package Signed-off-by: Esteve Fernandez <[email protected]> * fix: fix paths for downloaded artifacts Signed-off-by: Esteve Fernandez <[email protected]> * style(pre-commit): autofix * fix: create directories before downloading models Signed-off-by: Esteve Fernandez <[email protected]> --------- Signed-off-by: Esteve Fernandez <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
autowarefoundation · Sep 19, 2023 · f768b2c · f768b2c
1 parent 0e5ec31
commit f768b2c
Show file tree

Hide file tree

Showing 5 changed files with 360 additions and 0 deletions.
diff --git a/ansible/playbooks/universe.yaml b/ansible/playbooks/universe.yaml
@@ -7,6 +7,11 @@
         [Warning] Some Autoware components depend on the CUDA, cuDNN and TensorRT NVIDIA libraries which have end-user license agreements that should be reviewed before installation.
         Install NVIDIA libraries? [y/N]
       private: false
+    - name: prompt_download_artifacts
+      prompt: |-
+        [Warning] Should the ONNX model files and other artifacts be downloaded alongside setting up the development environment.
+        Download artifacts? [y/N]
+      private: false
   pre_tasks:
     - name: Verify OS
       ansible.builtin.fail:
@@ -49,3 +54,7 @@
     - role: autoware.dev_env.tensorrt
       when: prompt_install_nvidia == 'y'
     - role: autoware.dev_env.git_lfs
+
+    # ONNX files and other artifacts
+    - role: autoware.dev_env.artifacts
+      when: prompt_download_artifacts == 'y'
diff --git a/ansible/roles/artifacts/defaults/main.yaml b/ansible/roles/artifacts/defaults/main.yaml
diff --git a/ansible/roles/artifacts/meta/main.yaml b/ansible/roles/artifacts/meta/main.yaml
diff --git a/ansible/roles/artifacts/tasks/main.yaml b/ansible/roles/artifacts/tasks/main.yaml
@@ -0,0 +1,342 @@
+# yabloc_pose_initializer
+- name: Create yabloc_pose_initializer directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/yabloc_pose_initializer"
+    mode: "755"
+    state: directory
+
+- name: Download yabloc_pose_initializer/resources.tar.gz
+  become: true
+  ansible.builtin.get_url:
+    url: https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/136_road-segmentation-adas-0001/resources.tar.gz
+    dest: "{{ data_dir }}/yabloc_pose_initializer/resources.tar.gz"
+    mode: "644"
+    checksum: sha256:1f660e15f95074bade32b1f80dbf618e9cee1f0b9f76d3f4671cb9be7f56eb3a
+
+# image_projection_based_fusion
+- name: Create image_projection_based_fusion directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/image_projection_based_fusion"
+    mode: "755"
+    state: directory
+
+- name: Download image_projection_based_fusion/pts_voxel_encoder_pointpainting.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/pointpainting/v4/pts_voxel_encoder_pointpainting.onnx
+    dest: "{{ data_dir }}/image_projection_based_fusion/pts_voxel_encoder_pointpainting.onnx"
+    mode: "644"
+    checksum: md5:25c70f76a45a64944ccd19f604c99410
+
+- name: Download image_projection_based_fusion/pts_backbone_neck_head_pointpainting.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/pointpainting/v4/pts_backbone_neck_head_pointpainting.onnx
+    dest: "{{ data_dir }}/image_projection_based_fusion/pts_backbone_neck_head_pointpainting.onnx"
+    mode: "644"
+    checksum: md5:2c7108245240fbd7bf0104dd68225868
+
+# lidar_apollo_instance_segmentation
+- name: Create lidar_apollo_instance_segmentation directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/lidar_apollo_instance_segmentation"
+    mode: "755"
+    state: directory
+
+- name: Download lidar_apollo_instance_segmentation/vlp-16.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/vlp-16.onnx
+    dest: "{{ data_dir }}/lidar_apollo_instance_segmentation/vlp-16.onnx"
+    mode: "644"
+    checksum: md5:63a5a1bb73f7dbb64cf70d04eca45fb4
+
+- name: Download lidar_apollo_instance_segmentation/hdl-64.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/hdl-64.onnx
+    dest: "{{ data_dir }}/lidar_apollo_instance_segmentation/hdl-64.onnx"
+    mode: "644"
+    checksum: md5:009745e33b1df44b68296431cc384cd2
+
+- name: Download lidar_apollo_instance_segmentation/vls-128.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/vls-128.onnx
+    dest: "{{ data_dir }}/lidar_apollo_instance_segmentation/vls-128.onnx"
+    mode: "644"
+    checksum: md5:b2d709f56f73ae2518c9bf4d0214468f
+
+# lidar_centerpoint
+- name: Create lidar_centerpoint directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/lidar_centerpoint"
+    mode: "755"
+    state: directory
+
+- name: Download lidar_centerpoint/pts_voxel_encoder_centerpoint.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_voxel_encoder_centerpoint.onnx
+    dest: "{{ data_dir }}/lidar_centerpoint/pts_voxel_encoder_centerpoint.onnx"
+    mode: "644"
+    checksum: sha256:dc1a876580d86ee7a341d543f8ade2ede7f43bd032dc5b44155b1f0175405764
+
+- name: Download lidar_centerpoint/pts_backbone_neck_head_centerpoint.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_backbone_neck_head_centerpoint.onnx
+    dest: "{{ data_dir }}/lidar_centerpoint/pts_backbone_neck_head_centerpoint.onnx"
+    mode: "644"
+    checksum: sha256:3fe7e128955646740c41a25be0c8f141d5a94594fe79d7405fe2a859e391542e
+
+- name: Download lidar_centerpoint/pts_voxel_encoder_centerpoint_tiny.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_voxel_encoder_centerpoint_tiny.onnx
+    dest: "{{ data_dir }}/lidar_centerpoint/pts_voxel_encoder_centerpoint_tiny.onnx"
+    mode: "644"
+    checksum: sha256:2c53465715c1fd2e9dc5727ef3fca74f4cdf0538f74286b0946e219d0ca5693b
+
+- name: Download lidar_centerpoint/pts_backbone_neck_head_centerpoint_tiny.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_backbone_neck_head_centerpoint_tiny.onnx
+    dest: "{{ data_dir }}/lidar_centerpoint/pts_backbone_neck_head_centerpoint_tiny.onnx"
+    mode: "644"
+    checksum: md5:e4658325b70222f7c3637fe00e586b82
+
+# tensorrt_yolo
+- name: Create tensorrt_yolo directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/tensorrt_yolo"
+    mode: "755"
+    state: directory
+
+- name: Download tensorrt_yolo/yolov3.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov3.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov3.onnx"
+    mode: "644"
+    checksum: sha256:61e922f76918dd3d8e0abdc5fb7406f390609e08bd8ab9e5d3b97afb00f30f8c
+
+- name: Download tensorrt_yolo/yolov4.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov4.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov4.onnx"
+    mode: "644"
+    checksum: sha256:7c7343156c1bd4b397fd1e44b27334691a6219db3ce2e29a03b72af65ddb8f39
+
+- name: Download tensorrt_yolo/yolov4-tiny.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov4-tiny.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov4-tiny.onnx"
+    mode: "644"
+    checksum: sha256:0e877c716fbf8a2b431ee3e57f6c7411a6741319b52c32c6dafc53c7e1b17027
+
+- name: Download tensorrt_yolo/yolov5s.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov5s.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov5s.onnx"
+    mode: "644"
+    checksum: sha256:be335ff7746957debf1a6903a61fa3f568b780b4afe4958edf2d4bc98e9e0825
+
+- name: Download tensorrt_yolo/yolov5m.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov5m.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov5m.onnx"
+    mode: "644"
+    checksum: sha256:ee6f67f7c00a34cc4cef2fdd9db30dd714df1a4fb2d7e9fc1731cfe85b673133
+
+- name: Download tensorrt_yolo/yolov5l.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov5l.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov5l.onnx"
+    mode: "644"
+    checksum: sha256:a627e5f70180a8746482b572194090466db62c8d1256602c1cd20374dd960e34
+
+- name: Download tensorrt_yolo/yolov5x.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolov5x.onnx
+    dest: "{{ data_dir }}/tensorrt_yolo/yolov5x.onnx"
+    mode: "644"
+    checksum: sha256:d7cb4cd7078f87bda22a37828d72867accecedf9f74d0d87b5cc1f6f1180a019
+
+- name: Download tensorrt_yolo/coco.names
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/coco.names
+    dest: "{{ data_dir }}/tensorrt_yolo/coco.names"
+    mode: "644"
+    checksum: sha256:634a1132eb33f8091d60f2c346ababe8b905ae08387037aed883953b7329af84
+
+# tensorrt_yolox
+- name: Create tensorrt_yolox directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/tensorrt_yolox"
+    mode: "755"
+    state: directory
+
+- name: Download tensorrt_yolox/yolox-tiny.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolox-tiny.onnx
+    dest: "{{ data_dir }}/tensorrt_yolox/yolox-tiny.onnx"
+    mode: "644"
+    checksum: sha256:471a665f4243e654dff62578394e508db22ee29fe65d9e389dfc3b0f2dee1255
+
+- name: Download tensorrt_yolox/yolox-sPlus-opt.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolox-sPlus-opt.onnx
+    dest: "{{ data_dir }}/tensorrt_yolox/yolox-sPlus-opt.onnx"
+    mode: "644"
+    checksum: md5:bf3b0155351f90fcdca2626acbfd3bcf
+
+- name: Download tensorrt_yolox/yolox-sPlus-opt.EntropyV2-calibration.table
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/yolox-sPlus-opt.EntropyV2-calibration.table
+    dest: "{{ data_dir }}/tensorrt_yolox/yolox-sPlus-opt.EntropyV2-calibration.table"
+    mode: "644"
+    checksum: md5:c6e6f1999d5724a017516a956096701f
+
+- name: Download tensorrt_yolox/label.txt
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/label.txt
+    dest: "{{ data_dir }}/tensorrt_yolox/label.txt"
+    mode: "644"
+    checksum: sha256:3540a365bfd6d8afb1b5d8df4ec47f82cb984760d3270c9b41dbbb3422d09a0c
+
+# traffic_light_classifier
+- name: Create traffic_light_classifier directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/traffic_light_classifier"
+    mode: "755"
+    state: directory
+
+- name: Download traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_1.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_1.onnx
+    dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_1.onnx"
+    mode: "644"
+    checksum: md5:caa51f2080aa2df943e4f884c41898eb
+
+- name: Download traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_4.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_4.onnx
+    dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_4.onnx"
+    mode: "644"
+    checksum: md5:c2beaf60210f471debfe72b86d076ca0
+
+- name: Download traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_6.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_6.onnx
+    dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_6.onnx"
+    mode: "644"
+    checksum: md5:28b408710bcb24f4cdd4d746301d4e78
+
+- name: Download traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_1.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_1.onnx
+    dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_1.onnx"
+    mode: "644"
+    checksum: md5:82baba4fcf1abe0c040cd55005e34510
+
+- name: Download traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_4.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_4.onnx
+    dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_4.onnx"
+    mode: "644"
+    checksum: md5:21b549c2fe4fbb20d32cc019e6d70cd7
+
+- name: Download traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_6.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_6.onnx
+    dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_6.onnx"
+    mode: "644"
+    checksum: md5:378526d9aa9fc6705cf399f7b35b3053
+
+- name: Download traffic_light_classifier/lamp_labels.txt
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/lamp_labels.txt
+    dest: "{{ data_dir }}/traffic_light_classifier/lamp_labels.txt"
+    mode: "644"
+    checksum: sha256:1a5a49eeec5593963eab8d70f48b8a01bfb07e753e9688eb1510ad26e803579d
+
+# traffic_light_fine_detector
+- name: Create traffic_light_fine_detector directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/traffic_light_fine_detector"
+    mode: "755"
+    state: directory
+
+- name: Download traffic_light_fine_detector/tlr_yolox_s_batch_1.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_yolox_s_batch_1.onnx
+    dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_yolox_s_batch_1.onnx"
+    mode: "644"
+    checksum: md5:2b72d085022b8ee6aacff06bd722cfda
+
+- name: Download traffic_light_fine_detector/tlr_yolox_s_batch_4.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_yolox_s_batch_4.onnx
+    dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_yolox_s_batch_4.onnx"
+    mode: "644"
+    checksum: md5:4044daa86e7776ce241e94d98a09cc0e
+
+- name: Download traffic_light_fine_detector/tlr_yolox_s_batch_6.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_yolox_s_batch_6.onnx
+    dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_yolox_s_batch_6.onnx"
+    mode: "644"
+    checksum: md5:47255a11bde479320d703f1f45db1242
+
+- name: Download traffic_light_fine_detector/tlr_labels.txt
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_labels.txt
+    dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_labels.txt"
+    mode: "644"
+    checksum: md5:e9f45efb02f2a9aa8ac27b3d5c164905
+
+# traffic_light_ssd_fine_detector
+- name: Create traffic_light_ssd_fine_detector directory inside {{ data_dir }}
+  ansible.builtin.file:
+    path: "{{ data_dir }}/traffic_light_ssd_fine_detector"
+    mode: "755"
+    state: directory
+
+- name: Download traffic_light_ssd_fine_detector/mb2-ssd-lite-tlr.onnx
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/mb2-ssd-lite-tlr.onnx
+    dest: "{{ data_dir }}/traffic_light_ssd_fine_detector/mb2-ssd-lite-tlr.onnx"
+    mode: "644"
+    checksum: sha256:e29e6ee68751a270fb285fd037713939ca7f61a897b4c3a7ab22b0d6a9a21ddf
+
+- name: Download traffic_light_ssd_fine_detector/voc_labels_tl.txt
+  become: true
+  ansible.builtin.get_url:
+    url: https://awf.ml.dev.web.auto/perception/models/voc_labels_tl.txt
+    dest: "{{ data_dir }}/traffic_light_ssd_fine_detector/voc_labels_tl.txt"
+    mode: "644"
+    checksum: sha256:a41e6e3324e32c30b3b2fe38908eaf3471e2bfdaeb9e14ca0c1c3bc0275119c6
diff --git a/setup-dev-env.sh b/setup-dev-env.sh
@@ -9,6 +9,8 @@ SCRIPT_DIR=$(readlink -f "$(dirname "$0")")
 
 # Parse arguments
 args=()
+option_data_dir="$HOME/autoware_data"
+
 while [ "$1" != "" ]; do
     case "$1" in
     -y)
@@ -31,6 +33,11 @@ while [ "$1" != "" ]; do
         # Disable installation dev package of role 'cuda' and 'tensorrt'.
         option_runtime=true
         ;;
+    --data-dir)
+        # Set data directory
+        option_data_dir="$2"
+        shift
+        ;;
     *)
         args+=("$1")
         ;;
@@ -90,6 +97,8 @@ else
     ansible_args+=("--extra-vars" "install_devel=true")
 fi
 
+ansible_args+=("--extra-vars" "data_dir=$option_data_dir")
+
 # Load env
 source "$SCRIPT_DIR/amd64.env"
 if [ "$(uname -m)" = "aarch64" ]; then