Merge pull request #652 from roboflow/feature/e2e_tests_for_describe_…

…workflow_and_examples

E2E tests for describe workflow and examples

.github/workflows/test.jetson_4.5.0.yml

@@ -37,7 +37,7 @@ jobs:
           PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-jetson-4.5.0 make start_test_docker_jetson
       - name: 🧪 Regression Tests - Jetson 4.5.0
         run: |
-          SKIP_VISUALISATION_TESTS=true MAX_WAIT=300 SKIP_LMM_TEST=True SKIP_GROUNDING_DINO_TEST=true SKIP_SAM_TEST=true SKIP_YOLOV8_TEST=true SKIP_GAZE_TEST=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
+          SKIP_VISUALISATION_TESTS=true MAX_WAIT=300 SKIP_LMM_TEST=True SKIP_TROCR_TEST=True SKIP_GROUNDING_DINO_TEST=true SKIP_SAM_TEST=true SKIP_YOLOV8_TEST=true SKIP_GAZE_TEST=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
       - name: 🧹 Cleanup Test Docker - Jetson 4.5.0
         run: make stop_test_docker
         if: success() || failure()

.github/workflows/test.jetson_4.6.1.yml

@@ -37,7 +37,7 @@ jobs:
           PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-jetson-4.6.1 make start_test_docker_jetson
       - name: 🧪 Regression Tests - Jetson 4.6.1
         run: |
-          SKIP_YOLO_WORLD_TEST=true SKIP_SPEED_TEST=true SKIP_LMM_TEST=True SKIP_DOCTR_TEST=true SKIP_CLIP_TEST=true SKIP_VISUALISATION_TESTS=true MAX_WAIT=300 SKIP_SAM_TEST=true SKIP_GROUNDING_DINO_TEST=true SKIP_YOLOV8_TEST=true SKIP_GAZE_TEST=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
+          SKIP_YOLO_WORLD_TEST=true SKIP_SPEED_TEST=true SKIP_LMM_TEST=True SKIP_TROCR_TEST=True SKIP_TROCR_TEST=True SKIP_DOCTR_TEST=true SKIP_CLIP_TEST=true SKIP_VISUALISATION_TESTS=true MAX_WAIT=300 SKIP_SAM_TEST=true SKIP_GROUNDING_DINO_TEST=true SKIP_YOLOV8_TEST=true SKIP_GAZE_TEST=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
       - name: 🧹 Cleanup Test Docker - Jetson 4.6.1
         run: make stop_test_docker
         if: success() || failure()

development/stream_interface/objects_passing_line_demo.py

@@ -0,0 +1,125 @@
+import os
+from threading import Thread
+from typing import List, Optional, Union
+
+import cv2
+import supervision as sv
+
+from inference import InferencePipeline
+from inference.core.interfaces.camera.entities import VideoFrame
+from inference.core.interfaces.stream.watchdog import PipelineWatchDog, BasePipelineWatchDog
+from inference.core.utils.drawing import create_tiles
+
+STOP = False
+
+TIME_IN_ZONE_WORKFLOW = {
+    "version": "1.0",
+    "inputs": [
+        {"type": "WorkflowImage", "name": "image"},
+        {"type": "WorkflowVideoMetadata", "name": "video_metadata"},
+        {"type": "WorkflowParameter", "name": "line"},
+    ],
+    "steps": [
+        {
+            "type": "ObjectDetectionModel",
+            "name": "people_detector",
+            "image": "$inputs.image",
+            "model_id": "yolov8n-640",
+            "confidence": 0.6,
+        },
+        {
+            "type": "roboflow_core/byte_tracker@v1",
+            "name": "byte_tracker",
+            "detections": "$steps.people_detector.predictions",
+            "metadata": "$inputs.video_metadata"
+        },
+        {
+            "type": "roboflow_core/line_counter@v1",
+            "name": "line_counter",
+            "detections": f"$steps.byte_tracker.tracked_detections",
+            "metadata": "$inputs.video_metadata",
+            "line_segment": "$inputs.line",
+            "image": "$inputs.image",
+        },
+        {
+            "type": "roboflow_core/label_visualization@v1",
+            "name": "label_visualization",
+            "image": "$inputs.image",
+            "predictions": "$steps.byte_tracker.tracked_detections",
+        },
+        {
+            "type": "roboflow_core/bounding_box_visualization@v1",
+            "name": "bbox_visualization",
+            "image": "$steps.label_visualization.image",
+            "predictions": "$steps.byte_tracker.tracked_detections",
+        },
+        {
+            "type": "roboflow_core/line_counter_visualization@v1",
+            "name": "zone_visualization",
+            "image": "$steps.bbox_visualization.image",
+            "zone": "$inputs.line",
+            "count_in": "$steps.line_counter.count_in",
+            "count_out": "$steps.line_counter.count_out"
+        }
+    ],
+    "outputs": [
+        {"type": "JsonField", "name": "label_visualization", "selector": "$steps.zone_visualization.image"},
+    ],
+}
+
+
+def main() -> None:
+    global STOP
+    watchdog = BasePipelineWatchDog()
+    pipeline = InferencePipeline.init_with_workflow(
+        video_reference=os.environ["VIDEO_REFERENCE"],
+        workflow_specification=TIME_IN_ZONE_WORKFLOW,
+        watchdog=watchdog,
+        on_prediction=workflows_sink,
+        workflows_parameters={
+            "line": [[128, 512], [1900, 512]],
+        }
+    )
+    control_thread = Thread(target=command_thread, args=(pipeline, watchdog))
+    control_thread.start()
+    pipeline.start()
+    STOP = True
+    pipeline.join()
+
+
+def command_thread(pipeline: InferencePipeline, watchdog: PipelineWatchDog) -> None:
+    global STOP
+    while not STOP:
+        key = input()
+        if key == "i":
+            print(watchdog.get_report())
+        if key == "t":
+            pipeline.terminate()
+            STOP = True
+        elif key == "p":
+            pipeline.pause_stream()
+        elif key == "m":
+            pipeline.mute_stream()
+        elif key == "r":
+            pipeline.resume_stream()
+
+
+def workflows_sink(
+    predictions: Union[Optional[dict], List[Optional[dict]]],
+    video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]],
+) -> None:
+    images_to_show = []
+    if not isinstance(predictions, list):
+        predictions = [predictions]
+        video_frames = [video_frames]
+    for prediction, frame in zip(predictions, video_frames):
+        if prediction is None or frame is None:
+            continue
+        images_to_show.append(prediction["label_visualization"].numpy_image)
+    tiles = create_tiles(images=images_to_show)
+    cv2.imshow(f"Predictions", tiles)
+    cv2.waitKey(1)
+
+
+if __name__ == '__main__':
+    main()

development/stream_interface/time_in_zone_demo.py

@@ -47,10 +47,22 @@
             "image": "$inputs.image",
             "predictions": "$steps.time_in_zone.timed_detections",
             "text": "Time In Zone",
+        },
+        {
+            "type": "roboflow_core/bounding_box_visualization@v1",
+            "name": "bbox_visualization",
+            "image": "$steps.label_visualization.image",
+            "predictions": "$steps.time_in_zone.timed_detections",
+        },
+        {
+            "type": "roboflow_core/polygon_zone_visualization@v1",
+            "name": "zone_visualization",
+            "image": "$steps.bbox_visualization.image",
+            "zone": "$inputs.zone",
         }
     ],
     "outputs": [
-        {"type": "JsonField", "name": "label_visualization", "selector": "$steps.label_visualization.image"},
+        {"type": "JsonField", "name": "label_visualization", "selector": "$steps.zone_visualization.image"},
     ],
 }
 
@@ -64,7 +76,7 @@ def main() -> None:
         watchdog=watchdog,
         on_prediction=workflows_sink,
         workflows_parameters={
-            "zone": [(0, 0), (1000, 0), (1000, 1000), (0, 1000)],
+            "zone": [(0, 0), (512, 0), (512, 2000), (0, 2000)],
         }
     )
     control_thread = Thread(target=command_thread, args=(pipeline, watchdog))

tests/inference/hosted_platform_tests/conftest.py

@@ -2,7 +2,7 @@
 import os
 from enum import Enum
 from functools import partial
-from typing import Any
+from typing import Any, Tuple
 
 import numpy as np
 import pytest
@@ -77,6 +77,14 @@ class PlatformEnvironment(Enum):
     PlatformEnvironment.ROBOFLOW_STAGING: "coin-counting",
 }
 
+INTERFACE_DISCOVERING_WORKFLOW = {
+    PlatformEnvironment.ROBOFLOW_STAGING: ("paul-guerrie", "staging-test-workflow"),
+    PlatformEnvironment.ROBOFLOW_PLATFORM: (
+        "paul-guerrie-tang1",
+        "prod-test-workflow",
+    ),
+}
+
 ROBOFLOW_API_KEY = os.environ["HOSTED_PLATFORM_TESTS_API_KEY"]
 OPENAI_KEY = os.getenv("OPENAI_KEY")
 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
@@ -133,6 +141,13 @@ def target_project(platform_environment: PlatformEnvironment) -> str:
     return TARGET_PROJECTS_TO_BE_USED[platform_environment]
 
 
+@pytest.fixture(scope="session")
+def interface_discovering_workflow(
+    platform_environment: PlatformEnvironment,
+) -> Tuple[str, str]:
+    return INTERFACE_DISCOVERING_WORKFLOW[platform_environment]
+
+
 @pytest.fixture(scope="session", autouse=True)
 def warm_up_classification_lambda(
     classification_service_url: str,

tests/inference/hosted_platform_tests/test_workflows.py

@@ -1,3 +1,5 @@
+from typing import Tuple
+
 import pytest
 import requests
 
@@ -811,3 +813,169 @@ def test_getting_block_schema_using_get_endpoint(
     ), "Response expected to define required schema properties"
     assert "title" in schema, "Response expected to define unique schema title"
     assert "type" in schema, "Response expected to define schema type"
+
+
+@pytest.mark.flaky(retries=4, delay=1)
+def test_discovering_interface_of_saved_workflow(
+    object_detection_service_url: str,
+    interface_discovering_workflow: Tuple[str, str],
+) -> None:
+    # when
+    workspace_name, workflow_id = interface_discovering_workflow
+    result = requests.post(
+        f"{object_detection_service_url}/{workspace_name}/workflows/{workflow_id}/describe_interface",
+        json={"api_key": ROBOFLOW_API_KEY},
+    )
+
+    # then
+    result.raise_for_status()
+    response_data = result.json()
+    assert response_data["inputs"] == {
+        "image": ["image"],
+        "model_id": ["roboflow_model_id"],
+    }
+    assert response_data["outputs"] == {
+        "model_predictions": {
+            "predictions": ["object_detection_prediction"],
+            "inference_id": ["string"],
+        },
+        "bounding_box_visualization": ["image"],
+    }
+    assert response_data["typing_hints"] == {
+        "image": "dict",
+        "object_detection_prediction": "dict",
+        "string": "str",
+        "roboflow_model_id": "str",
+    }
+    assert set(response_data["kinds_schemas"].keys()) == {
+        "image",
+        "object_detection_prediction",
+    }
+
+
+@pytest.mark.flaky(retries=4, delay=1)
+def test_discovering_interface_of_valid_workflow_from_payload(
+    object_detection_service_url: str,
+) -> None:
+    # given
+    valid_definition = {
+        "version": "1.0.0",
+        "inputs": [
+            {"type": "WorkflowImage", "name": "image"},
+            {"type": "WorkflowParameter", "name": "model_id"},
+            {"type": "WorkflowParameter", "name": "confidence"},
+        ],
+        "steps": [
+            {
+                "type": "ObjectDetectionModel",
+                "name": "general_detection_1",
+                "image": "$inputs.image",
+                "model_id": "$inputs.model_id",
+                "class_filter": ["dog"],
+            },
+            {
+                "type": "ObjectDetectionModel",
+                "name": "general_detection_2",
+                "image": "$inputs.image",
+                "model_id": "$inputs.model_id",
+                "confidence": "$inputs.confidence",
+                "class_filter": ["dog"],
+            },
+        ],
+        "outputs": [
+            {
+                "type": "JsonField",
+                "name": "detections",
+                "selector": "$steps.general_detection_1.predictions",
+            },
+            {
+                "type": "JsonField",
+                "name": "detections",
+                "selector": "$steps.general_detection_2.predictions",
+            },
+        ],
+    }
+
+    # when
+    response = requests.post(
+        f"{object_detection_service_url}/workflows/describe_interface",
+        json={
+            "specification": valid_definition,
+            "api_key": "some",
+        },
+    )
+
+    # then
+    response.raise_for_status()
+    response_data = response.json()
+    assert response_data["outputs"] == {"detections": ["object_detection_prediction"]}
+    assert response_data["inputs"] == {
+        "image": ["image"],
+        "model_id": ["roboflow_model_id"],
+        "confidence": ["float_zero_to_one"],
+    }
+    assert response_data["typing_hints"] == {
+        "float_zero_to_one": "float",
+        "image": "dict",
+        "object_detection_prediction": "dict",
+        "roboflow_model_id": "str",
+    }
+    assert set(response_data["kinds_schemas"].keys()) == {
+        "object_detection_prediction",
+        "image",
+    }, "Expected image and object_detection_prediction kinds to deliver schema"
+
+
+@pytest.mark.flaky(retries=4, delay=1)
+def test_discovering_interface_of_invalid_workflow_from_payload(
+    object_detection_service_url: str,
+) -> None:
+    # given
+    valid_definition = {
+        "version": "1.0.0",
+        "inputs": [
+            {"type": "WorkflowImage", "name": "image"},
+            {"type": "WorkflowParameter", "name": "model_id"},
+        ],
+        "steps": [
+            {
+                "type": "ObjectDetectionModel",
+                "name": "general_detection_1",
+                "image": "$inputs.image",
+                "model_id": "$inputs.model_id",
+                "class_filter": ["dog"],
+            },
+            {
+                "type": "ObjectDetectionModel",
+                "name": "general_detection_2",
+                "image": "$inputs.image",
+                "model_id": "$inputs.model_id",
+                "confidence": "$inputs.model_id",
+                "class_filter": ["dog"],
+            },
+        ],
+        "outputs": [
+            {
+                "type": "JsonField",
+                "name": "detections",
+                "selector": "$steps.general_detection_1.predictions",
+            },
+            {
+                "type": "JsonField",
+                "name": "detections",
+                "selector": "$steps.general_detection_2.predictions",
+            },
+        ],
+    }
+
+    # when
+    response = requests.post(
+        f"{object_detection_service_url}/workflows/describe_interface",
+        json={
+            "specification": valid_definition,
+            "api_key": "some",
+        },
+    )
+
+    # then
+    assert response.status_code == 400