Skip to content

Commit

Permalink
tests for keras_preprocess
Browse files Browse the repository at this point in the history
Signed-off-by: Abhishek Gaikwad <[email protected]>
  • Loading branch information
gaikwadabhishek committed Jul 29, 2023
1 parent 160b38d commit 08bfd27
Show file tree
Hide file tree
Showing 12 changed files with 98 additions and 13 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/transformer-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ on:
required: true
type: boolean
default: true
build_keras_preprocess_image:
description: 'Build Keras Preprocess Image (aistore/transformer_keras:latest)'
required: true
type: boolean
default: true
push:
paths:
- 'transformers/**'
Expand All @@ -44,6 +49,7 @@ env:
MD5_ENABLE: ${{ github.event.inputs.build_md5_image }}
TAR2TF_ENABLE: ${{ github.event.inputs.build_tar2tf_image }}
COMPRESS_ENABLE: ${{ github.event.inputs.build_compress_image }}
KERAS_ENABLE: ${{ github.event.inputs.build_keras_preprocess_image }}

jobs:
build:
Expand Down
6 changes: 5 additions & 1 deletion transformers/Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
SUBDIRS := compress echo go_echo hello_world md5 tar2tf
SUBDIRS := compress echo go_echo hello_world md5 tar2tf keras_preprocess

ifeq ($(ECHO_ENABLE), false)
SUBDIRS := $(filter-out echo,$(SUBDIRS))
Expand All @@ -24,6 +24,10 @@ ifeq ($(COMPRESS_ENABLE), false)
SUBDIRS := $(filter-out compress,$(SUBDIRS))
endif

ifeq ($(KERAS_ENABLE), false)
SUBDIRS := $(filter-out compress,$(SUBDIRS))
endif

all: $(SUBDIRS)
$(SUBDIRS):
$(MAKE) -C $@ all
Expand Down
File renamed without changes.
15 changes: 15 additions & 0 deletions transformers/keras_preprocess/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Default image tag is 'latest'
TAG := latest
ifeq ($(GIT_TEST), true)
TAG := test
endif

REGISTRY_URL ?= docker.io/aistorage

all: build push

build:
docker build -t $(REGISTRY_URL)/transformer_keras:$(TAG) .

push:
docker push $(REGISTRY_URL)/transformer_keras:$(TAG)
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ metadata:
spec:
containers:
- name: server
image: gaikwadabhishek/transformer_keras:latest
image: aistorage/transformer_keras:latest
imagePullPolicy: Always
ports:
- name: default
Expand Down
File renamed without changes.
9 changes: 0 additions & 9 deletions transformers/keras_transformer/Makefile

This file was deleted.

6 changes: 5 additions & 1 deletion transformers/tests/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,8 @@ numpy
pillow
pyyaml
scikit-image
tensorflow
tensorflow
requests
scipy
keras
pyyaml
2 changes: 1 addition & 1 deletion transformers/tests/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
class TestBase(unittest.TestCase):
def setUp(self):
self.endpoint = os.environ.get("AIS_ENDPOINT", "http://192.168.49.2:8080")
self.git_test_mode = os.getenv('GIT_TEST', 'False')
self.git_test_mode = os.getenv('GIT_TEST', 'false')
self.client = Client(self.endpoint)
self.test_bck = self.client.bucket("etl-test-bucket").create(exist_ok=True)
self.test_etl = self.client.etl("test-etl-" + generate_random_str())
Expand Down
65 changes: 65 additions & 0 deletions transformers/tests/test_keras_transformer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#
# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
#

# pylint: disable=missing-class-docstring, missing-function-docstring, missing-module-docstring

import unittest
import io
import os
from aistore.sdk.etl_const import ETL_COMM_HPULL
from aistore.sdk.etl_templates import KERAS_TRANSFORMER

from keras.preprocessing.image import (
ImageDataGenerator,
load_img,
array_to_img,
img_to_array,
)
from test_base import TestBase
from utils import git_test_mode_format_image_tag_test


class TestTransformers(TestBase):
def setUp(self):
super().setUp()
self.test_image_filename = "test-image.jpg"
self.test_image_source = "./resources/test-image.jpg"
self.test_bck.object(self.test_image_filename).put_file(self.test_image_source)

@unittest.skipIf(
os.getenv("KERAS_ENABLE", "true") == "false",
"Keras image was not built, skipping keras test",
)
def test_keras_transformer(self):
template = KERAS_TRANSFORMER.format(
communication_type=ETL_COMM_HPULL,
format="JPEG",
transform='{"theta":40, "brightness":0.8, "zx":0.9, "zy":0.9}',
)
if self.git_test_mode == "true":
template = git_test_mode_format_image_tag_test(template, "keras")

self.test_etl.init_spec(template=template)

# transformed image - etl
transformed_image_etl = (
self.test_bck.object(self.test_image_filename)
.get(etl_name=self.test_etl.name)
.read_all()
)

# transformed image - local
img = load_img(self.test_image_source)
img = img_to_array(img)
datagen = ImageDataGenerator()
rotate = datagen.apply_transform(
x=img,
transform_parameters={"theta": 40, "brightness": 0.8, "zx": 0.9, "zy": 0.9},
)
img = array_to_img(rotate)
buf = io.BytesIO()
img.save(buf, format="JPEG")
transformed_image_local = buf.getvalue()

self.assertEqual(transformed_image_local, transformed_image_etl)

0 comments on commit 08bfd27

Please sign in to comment.