tests for keras_preprocess

Signed-off-by: Abhishek Gaikwad <[email protected]>

.github/workflows/transformer-docker.yml

@@ -33,6 +33,11 @@ on:
         required: true
         type: boolean
         default: true
+      build_keras_preprocess_image:
+        description: 'Build Keras Preprocess Image (aistore/transformer_keras:latest)'
+        required: true
+        type: boolean
+        default: true
   push:
     paths:
       - 'transformers/**'
@@ -44,6 +49,7 @@ env:
   MD5_ENABLE: ${{ github.event.inputs.build_md5_image }}
   TAR2TF_ENABLE: ${{ github.event.inputs.build_tar2tf_image }}
   COMPRESS_ENABLE: ${{ github.event.inputs.build_compress_image }}
+  KERAS_ENABLE: ${{ github.event.inputs.build_keras_preprocess_image }}
 
 jobs:
   build:

transformers/Makefile

@@ -1,4 +1,4 @@
-SUBDIRS := compress echo go_echo hello_world md5 tar2tf
+SUBDIRS := compress echo go_echo hello_world md5 tar2tf keras_preprocess
 
 ifeq ($(ECHO_ENABLE), false)
 SUBDIRS := $(filter-out echo,$(SUBDIRS))
@@ -24,6 +24,10 @@ ifeq ($(COMPRESS_ENABLE), false)
 SUBDIRS := $(filter-out compress,$(SUBDIRS))
 endif
 
+ifeq ($(KERAS_ENABLE), false)
+SUBDIRS := $(filter-out compress,$(SUBDIRS))
+endif
+
 all: $(SUBDIRS)
 $(SUBDIRS):
 	$(MAKE) -C $@ all

transformers/keras_preprocess/Makefile

@@ -0,0 +1,15 @@
+# Default image tag is 'latest'
+TAG := latest
+ifeq ($(GIT_TEST), true)
+	TAG := test
+endif
+
+REGISTRY_URL ?= docker.io/aistorage
+
+all: build push
+
+build:
+	docker build -t $(REGISTRY_URL)/transformer_keras:$(TAG) .
+
+push:
+	docker push $(REGISTRY_URL)/transformer_keras:$(TAG)

transformers/keras_transformer/pod.yaml → transformers/keras_preprocess/pod.yaml

@@ -11,7 +11,7 @@ metadata:
 spec:
   containers:
     - name: server
-      image: gaikwadabhishek/transformer_keras:latest
+      image: aistorage/transformer_keras:latest
       imagePullPolicy: Always
       ports:
         - name: default

transformers/tests/requirements.txt

@@ -3,4 +3,8 @@ numpy
 pillow
 pyyaml
 scikit-image
-tensorflow
+tensorflow
+requests
+scipy
+keras
+pyyaml

transformers/tests/test_base.py

@@ -12,7 +12,7 @@
 class TestBase(unittest.TestCase):
     def setUp(self):
         self.endpoint = os.environ.get("AIS_ENDPOINT", "http://192.168.49.2:8080")
-        self.git_test_mode = os.getenv('GIT_TEST', 'False')
+        self.git_test_mode = os.getenv('GIT_TEST', 'false')
         self.client = Client(self.endpoint)
         self.test_bck = self.client.bucket("etl-test-bucket").create(exist_ok=True)
         self.test_etl = self.client.etl("test-etl-" + generate_random_str()) 

transformers/tests/test_keras_transformer.py

@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+#
+
+# pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring
+
+import unittest
+import io
+import os
+from aistore.sdk.etl_const import ETL_COMM_HPULL
+from aistore.sdk.etl_templates import KERAS_TRANSFORMER
+
+from keras.preprocessing.image import (
+    ImageDataGenerator,
+    load_img,
+    array_to_img,
+    img_to_array,
+)
+from test_base import TestBase
+from utils import git_test_mode_format_image_tag_test
+
+
+class TestTransformers(TestBase):
+    def setUp(self):
+        super().setUp()
+        self.test_image_filename = "test-image.jpg"
+        self.test_image_source = "./resources/test-image.jpg"
+        self.test_bck.object(self.test_image_filename).put_file(self.test_image_source)
+
+    @unittest.skipIf(
+        os.getenv("KERAS_ENABLE", "true") == "false",
+        "Keras image was not built, skipping keras test",
+    )
+    def test_keras_transformer(self):
+        template = KERAS_TRANSFORMER.format(
+            communication_type=ETL_COMM_HPULL,
+            format="JPEG",
+            transform='{"theta":40, "brightness":0.8, "zx":0.9, "zy":0.9}',
+        )
+        if self.git_test_mode == "true":
+            template = git_test_mode_format_image_tag_test(template, "keras")
+
+        self.test_etl.init_spec(template=template)
+
+        # transformed image - etl
+        transformed_image_etl = (
+            self.test_bck.object(self.test_image_filename)
+            .get(etl_name=self.test_etl.name)
+            .read_all()
+        )
+
+        # transformed image - local
+        img = load_img(self.test_image_source)
+        img = img_to_array(img)
+        datagen = ImageDataGenerator()
+        rotate = datagen.apply_transform(
+            x=img,
+            transform_parameters={"theta": 40, "brightness": 0.8, "zx": 0.9, "zy": 0.9},
+        )
+        img = array_to_img(rotate)
+        buf = io.BytesIO()
+        img.save(buf, format="JPEG")
+        transformed_image_local = buf.getvalue()
+
+        self.assertEqual(transformed_image_local, transformed_image_etl)