Import punet tests from sharktank to iree_tests.

.github/workflows/test_iree.yml

@@ -157,15 +157,15 @@ jobs:
  iree_tests/pytorch/models \
  iree_tests/sharktank \
  -n 4 \
- -rpfE \
+ -rA \
  -k real_weights \
  --no-skip-tests-missing-files \
  --capture=no \
  --log-cli-level=info \
  --timeout=1200 \
  --durations=0 \
  --config-files=${MODELS_CONFIG_FILE_PATH}
- 
+
  - name: "Running SDXL special model tests"
  id: special_models_sdxl
  if: ${{ !cancelled() }}
@@ -178,7 +178,7 @@ jobs:
  --log-cli-level=info \
  --timeout=1200 \
  --durations=0
- 
+
  - name: "Running SD3 special model tests"
  id: special_models_sd3
  if: ${{ !cancelled() }}

.gitignore

@@ -10,6 +10,10 @@
 *.venv
 .venv
 
+# Python deployment artifacts
+*.whl
+*.egg-info
+
 # Visual Studio files
 .vs/
 .vscode/

iree_tests/.gitignore

@@ -9,4 +9,5 @@ config_*.txt
 # Large files fetched on-demand
 *.bin
 real_weights.irpa
+*_dataset.irpa
 pytorch/models/**/*.npy

iree_tests/configs/gpu_cuda.json

@@ -0,0 +1,13 @@
+{
+ "config_name": "gpu_cuda",
+ "iree_compile_flags" : [
+ "--iree-hal-target-backends=cuda"
+ ],
+ "iree_run_module_flags": [
+ "--device=cuda"
+ ],
+ "skip_compile_tests": [],
+ "skip_run_tests": [],
+ "expected_compile_failures": [],
+ "expected_run_failures": []
+}

iree_tests/configs/gpu_vulkan.json

@@ -6,16 +6,8 @@
  "iree_run_module_flags": [
  "--device=vulkan"
  ],
- "skip_compile_tests": [
- // TODO(#16666): Fix these tests hanging at runtime (miscompiles?)
- "onnx/node/generated/test_dequantizelinear",
- "onnx/node/generated/test_slice_default_axes",
- ],
+ "skip_compile_tests": [],
  "skip_run_tests": [],
- "expected_compile_failures": [
- // Empty to keep this file as a simple demo.
- ],
- "expected_run_failures": [
- // Empty to keep this file as a simple demo.
- ]
+ "expected_compile_failures": [],
+ "expected_run_failures": []
 }

iree_tests/configs/models_cpu_llvm_task.json

@@ -14,8 +14,12 @@
  ],
  "skip_run_tests": [],
  "expected_compile_failures": [
- "pytorch/models/opt-125M", // TODO(#17344): need to regenerate .mlirbc
- "pytorch/models/resnet50", // TODO(#17344): need to regenerate .mlirbc
+ // TODO(#17344): need to regenerate .mlirbc
+ "pytorch/models/opt-125M",
+ "pytorch/models/resnet50",
+
+ // error: One or more operations with large vector sizes (16384 bytes) were found
+ "sharktank/punet/int8",
  ],
  "expected_run_failures": []
  }

iree_tests/configs/models_gpu_rocm_gfx90a.json

@@ -3,24 +3,23 @@
  "iree_compile_flags" : [
  "--iree-hal-target-backends=rocm",
  "--iree-rocm-target-chip=gfx90a",
- "--iree-opt-const-eval=false",
- "--iree-codegen-transform-dialect-library=${IREE_TEST_PATH_EXTENSION}/attention_and_matmul_spec.mlir"
  ],
  "iree_run_module_flags": [
  "--device=hip"
  ],
- "skip_compile_tests": [
- "pytorch/models/sdxl-scheduled-unet-3-tank",
- "pytorch/models/sdxl-vae-decode-tank",
- "pytorch/models/sdxl-prompt-encoder-tank"
- ],
+ "skip_compile_tests": [],
  "skip_run_tests": [],
  "expected_compile_failures": [
- "pytorch/models/opt-125M", // TODO(#17344): need to regenerate .mlirbc
+ // TODO(#17344): need to regenerate .mlirbc
+ "pytorch/models/opt-125M",
  "pytorch/models/resnet50",
- // error: 'builtin.module' op failed to run transform dialect passes
- // (might need to drop the iree-codegen-transform-dialect-library flag)
- "sharktank/llama/open-llama-3b-v2-f16"
+
+ // error: a handle passed as operand #0 and consumed by this operation points to a payload entity more than once
+ "sharktank/llama/open-llama-3b-v2-f16",
+
+ // failed to run translation of source executable to target executable for backend
+ "sharktank/punet/fp16",
+ "sharktank/punet/int8",
  ],
  "expected_run_failures": []
 }

iree_tests/sharktank/punet/fp16/real_weights_data_flags.txt

@@ -0,0 +1,7 @@
+--parameters=model=sdxl_fp16_dataset.irpa
+--input=1x4x128x128xf16
+--input=1xi32
+--input=2x64x2048xf16
+--input=2x1280xf16
+--input=2x6xf16
+--input=1xf16

iree_tests/sharktank/punet/fp16/test_cases.json

@@ -0,0 +1,13 @@
+{
+ "file_format": "test_cases_v0",
+ "test_cases": [
+ {
+ "name": "real_weights",
+ "runtime_flagfile": "real_weights_data_flags.txt",
+ "remote_files": [
+ "https://sharkpublic.blob.core.windows.net/sharkpublic/scotttodd/iree_tests/2024_07_02/sdxl_fp16_dataset.irpa",
+ // TODO: files for real inputs and real expected outputs
+ ]
+ }
+ ]
+}

iree_tests/sharktank/punet/int8/real_weights_data_flags.txt

@@ -0,0 +1,7 @@
+--parameters=model=sdxl_int8_dataset.irpa
+--input=1x4x128x128xf16
+--input=1xi32
+--input=2x64x2048xf16
+--input=2x1280xf16
+--input=2x6xf16
+--input=1xf16

iree_tests/sharktank/punet/int8/test_cases.json

@@ -0,0 +1,13 @@
+{
+ "file_format": "test_cases_v0",
+ "test_cases": [
+ {
+ "name": "real_weights",
+ "runtime_flagfile": "real_weights_data_flags.txt",
+ "remote_files": [
+ "https://sharkpublic.blob.core.windows.net/sharkpublic/scotttodd/iree_tests/2024_07_02/sdxl_int8_dataset.irpa",
+ // TODO: files for real inputs and real expected outputs
+ ]
+ }
+ ]
+}