implement API for EvaluationResult protocol

app/README.md

@@ -24,6 +24,5 @@ $ cd app
 $ sh drucker-grpc-proto/run_codegen.sh
 $ cp drucker-grpc-proto/protobuf/drucker_pb2.py .
 $ cp drucker-grpc-proto/protobuf/drucker_pb2_grpc.py .
-$ python -m unittest test/test_api_service.py
-$ python -m unittest test/test_api_application.py
+$ python -m unittest test/test_api_evaluation.py
 ```

app/apis/__init__.py

@@ -5,6 +5,7 @@
 from apis.api_application import app_info_namespace
 from apis.api_service import srv_info_namespace
 from apis.api_model import mdl_info_namespace
+from apis.api_evaluation import eval_info_namespace
 from apis.api_misc import misc_info_namespace
 from apis.api_admin import admin_info_namespace
 from auth import auth_required
@@ -40,5 +41,6 @@ def default_error_handler(error):
 api.add_namespace(app_info_namespace, path='/api/applications')
 api.add_namespace(srv_info_namespace, path='/api/applications')
 api.add_namespace(mdl_info_namespace, path='/api/applications')
+api.add_namespace(eval_info_namespace, path='/api/applications')
 api.add_namespace(admin_info_namespace, path='/api/applications')
 api.add_namespace(misc_info_namespace, path='/api')

app/apis/api_application.py

@@ -1,17 +1,18 @@
 import uuid
-import datetime
 
 from flask_jwt_simple import get_jwt_identity
 from flask_restplus import Namespace, fields, Resource, reqparse
-from werkzeug.datastructures import FileStorage
 
 from app import logger
 from auth import auth
 from models import db
+<<<<<<< HEAD
+from models import Application, Service
+=======
 from models import Application, Service, Evaluation, EvaluationResult, ApplicationUserRole, Role, User
+>>>>>>> master
 from core.drucker_dashboard_client import DruckerDashboardClient
 from apis.common import DatetimeToTimestamp
-from utils.hash_util import HashUtil
 
 
 app_info_namespace = Namespace('applications', description='Application Endpoint.')
@@ -145,58 +146,3 @@ def patch(self, application_id:int):
         db.session.commit()
         db.session.close()
         return response_body
-
-@app_info_namespace.route('/<int:application_id>/evaluation')
-class ApiEvaluation(Resource):
-    upload_parser = reqparse.RequestParser()
-    upload_parser.add_argument('file', location='files', type=FileStorage, required=True)
-
-    @app_info_namespace.expect(upload_parser)
-    def post(self, application_id:int):
-        """update data to be evaluated"""
-        args = self.upload_parser.parse_args()
-        file = args['file']
-        checksum = HashUtil.checksum(file)
-
-        eobj = db.session.query(Evaluation).filter(
-            Evaluation.application_id == application_id,
-            Evaluation.checksum == checksum).one_or_none()
-        if eobj is not None:
-            return {"status": True, "evaluation_id": eobj.evaluation_id}
-
-        eval_data_path = "eval-{0:%Y%m%d%H%M%S}.txt".format(datetime.datetime.utcnow())
-
-        sobj = Service.query.filter_by(application_id=application_id).first_or_404()
-
-        drucker_dashboard_application = DruckerDashboardClient(logger=logger, host=sobj.host)
-        response_body = drucker_dashboard_application.run_upload_evaluation_data(file, eval_data_path)
-
-        if not response_body['status']:
-            raise Exception('Failed to upload')
-        eobj = Evaluation(checksum=checksum, application_id=application_id, data_path=eval_data_path)
-        db.session.add(eobj)
-        db.session.flush()
-        evaluation_id = eobj.evaluation_id
-        db.session.commit()
-        db.session.close()
-
-        return {"status": True, "evaluation_id": evaluation_id}
-
-
-@app_info_namespace.route('/<int:application_id>/evaluation/<int:evaluation_id>')
-class ApiEvaluation(Resource):
-    def delete(self, application_id:int, evaluation_id:int):
-        """delete data to be evaluated"""
-        eval_query = db.session.query(Evaluation)\
-            .filter(Evaluation.application_id == application_id,
-                    Evaluation.evaluation_id == evaluation_id)
-        if eval_query.one_or_none() is None:
-            return {"status": False}, 404
-
-        eval_query.delete()
-        db.session.query(EvaluationResult)\
-            .filter(EvaluationResult.evaluation_id == evaluation_id).delete()
-        db.session.commit()
-        db.session.close()
-
-        return {"status": True, "message": "Success."}

app/apis/api_evaluation.py

@@ -0,0 +1,192 @@
+import datetime
+import json
+from itertools import chain
+
+from flask_restplus import Namespace, fields, Resource, reqparse
+from werkzeug.datastructures import FileStorage
+
+from app import logger
+from models import db
+from models import Service, Evaluation, EvaluationResult
+from core.drucker_dashboard_client import DruckerDashboardClient
+from utils.hash_util import HashUtil
+
+
+eval_info_namespace = Namespace('evaluation', description='Evaluation Endpoint.')
+success_or_not = eval_info_namespace.model('Success', {
+    'status': fields.Boolean(required=True),
+    'message': fields.String(required=True)
+})
+eval_metrics = eval_info_namespace.model('Evaluation result', {
+    'num': fields.Integer(required=True, description='number of evaluated data'),
+    'accuracy': fields.Float(required=True, description='accuracy of evaluation'),
+    'fvalue': fields.List(fields.Float, required=True, description='F-value of evaluation'),
+    'precision': fields.List(fields.Float, required=True, description='precision of evaluation'),
+    'recall': fields.List(fields.Float, required=True, description='recall of evaluation'),
+    'option': fields.Raw(),
+    'status': fields.Boolean(required=True),
+    'result_id': fields.Integer(required=True, description='ID of evaluation result')
+})
+eval_data_upload = eval_info_namespace.model('Result of uploading evaluation data', {
+    'status': fields.Boolean(required=True),
+    'evaluation_id': fields.Integer(required=True, description='ID of uploaded data')
+})
+
+
+@eval_info_namespace.route('/<int:application_id>/evaluation')
+class ApiEvaluation(Resource):
+    upload_parser = reqparse.RequestParser()
+    upload_parser.add_argument('file', location='files', type=FileStorage, required=True)
+
+    @eval_info_namespace.expect(upload_parser)
+    @eval_info_namespace.marshal_with(eval_data_upload)
+    def post(self, application_id:int):
+        """update data to be evaluated"""
+        args = self.upload_parser.parse_args()
+        file = args['file']
+        checksum = HashUtil.checksum(file)
+
+        eobj = db.session.query(Evaluation).filter(
+            Evaluation.application_id == application_id,
+            Evaluation.checksum == checksum).one_or_none()
+        if eobj is not None:
+            return {"status": True, "evaluation_id": eobj.evaluation_id}
+
+        eval_data_path = "eval-{0:%Y%m%d%H%M%S}.txt".format(datetime.datetime.utcnow())
+
+        sobj = Service.query.filter_by(application_id=application_id).first_or_404()
+
+        drucker_dashboard_application = DruckerDashboardClient(logger=logger, host=sobj.host)
+        response_body = drucker_dashboard_application.run_upload_evaluation_data(file, eval_data_path)
+
+        if not response_body['status']:
+            raise Exception('Failed to upload')
+        eobj = Evaluation(checksum=checksum, application_id=application_id, data_path=eval_data_path)
+        db.session.add(eobj)
+        db.session.flush()
+        evaluation_id = eobj.evaluation_id
+        db.session.commit()
+        db.session.close()
+
+        return {"status": True, "evaluation_id": evaluation_id}
+
+
+@eval_info_namespace.route('/<int:application_id>/evaluation/<int:evaluation_id>')
+class ApiEvaluation(Resource):
+
+    @eval_info_namespace.marshal_with(success_or_not)
+    def delete(self, application_id:int, evaluation_id:int):
+        """delete data to be evaluated"""
+        eval_query = db.session.query(Evaluation)\
+            .filter(Evaluation.application_id == application_id,
+                    Evaluation.evaluation_id == evaluation_id)
+        if eval_query.one_or_none() is None:
+            return {"status": False, "message": "Not Found."}, 404
+
+        eval_query.delete()
+        db.session.query(EvaluationResult)\
+            .filter(EvaluationResult.evaluation_id == evaluation_id).delete()
+        db.session.commit()
+        db.session.close()
+
+        return {"status": True, "message": "Success."}
+
+
+@eval_info_namespace.route('/<int:application_id>/evaluate')
+class ApiEvaluate(Resource):
+    eval_parser = reqparse.RequestParser()
+    eval_parser.add_argument('service_id', location='form', type=int, required=True)
+    eval_parser.add_argument('evaluation_id', location='form', type=int, required=False)
+    eval_parser.add_argument('overwrite', location='form', type=bool, required=False)
+
+    @eval_info_namespace.expect(eval_parser)
+    @eval_info_namespace.marshal_with(eval_metrics)
+    def post(self, application_id:int):
+        """evaluate"""
+        args = self.eval_parser.parse_args()
+        eval_id = args.get('evaluation_id', None)
+        service_id = args.get('service_id')
+        if eval_id:
+            eobj = Evaluation.query.filter_by(
+                application_id=application_id,
+                evaluation_id=eval_id).first_or_404()
+        else:
+            # if evaluation_id is not given, use the lastest one.
+            eobj = Evaluation.query\
+                .filter_by(application_id=application_id)\
+                .order_by(Evaluation.register_date.desc()).first_or_404()
+        sobj = Service.query.filter_by(
+            application_id=application_id,
+            service_id=service_id).first_or_404()
+
+        robj = db.session.query(EvaluationResult)\
+            .filter(EvaluationResult.model_id == sobj.model_id,
+                    EvaluationResult.evaluation_id == eobj.evaluation_id).one_or_none()
+        if robj is not None and args.get('overwrite', False):
+            return robj.result
+
+        eval_result_path = "eval-result-{0:%Y%m%d%H%M%S}.txt".format(datetime.datetime.utcnow())
+        drucker_dashboard_application = DruckerDashboardClient(logger=logger, host=sobj.host)
+        response_body = drucker_dashboard_application.run_evaluate_model(eobj.data_path, eval_result_path)
+
+        if response_body['status']:
+            result = json.dumps(response_body)
+            if robj is None:
+                robj = EvaluationResult(model_id=sobj.model_id,
+                                        data_path=eval_result_path,
+                                        evaluation_id=eobj.evaluation_id,
+                                        result=result)
+                db.session.add(robj)
+            else:
+                robj.data_path = eval_result_path
+                robj.result = result
+            db.session.flush()
+            response_body = robj.result
+            db.session.commit()
+            db.session.close()
+
+        return response_body
+
+
+@eval_info_namespace.route('/<int:application_id>/evaluation_result/<int:eval_result_id>')
+class ApiEvaluationResult(Resource):
+
+    def get(self, application_id:int, eval_result_id:int):
+        """get detailed evaluation result"""
+        eval_with_result = db.session.query(Evaluation, EvaluationResult)\
+            .filter(Evaluation.application_id == application_id,
+                    EvaluationResult.evaluation_id == Evaluation.evaluation_id,
+                    EvaluationResult.evaluation_result_id == eval_result_id).one_or_none()
+        if eval_with_result is None:
+            return {"status": False, "message": "Not Found."}, 404
+        sobj = Service.query.filter_by(application_id=application_id).first_or_404()
+        drucker_dashboard_application = DruckerDashboardClient(logger=logger, host=sobj.host)
+        eobj = eval_with_result.Evaluation
+        robj = eval_with_result.EvaluationResult
+
+        response_body = list(drucker_dashboard_application.run_evaluation_data(eobj.data_path, robj.data_path))
+        if len(response_body) == 0:
+            return {"status": False, "message": "Result Not Found."}, 404
+
+        return {
+            'status': all(r['status'] for r in response_body),
+            'metrics': response_body[0]['metrics'],
+            'details': list(chain.from_iterable(r['detail'] for r in response_body))
+        }
+
+    @eval_info_namespace.marshal_with(success_or_not)
+    def delete(self, application_id:int, eval_result_id:int):
+        """get detailed evaluation result"""
+        eval_with_result = db.session.query(Evaluation, EvaluationResult)\
+            .filter(Evaluation.application_id == application_id,
+                    EvaluationResult.evaluation_id == Evaluation.evaluation_id,
+                    EvaluationResult.evaluation_result_id == eval_result_id).one_or_none()
+        if eval_with_result is None:
+            return {"status": False, "message": "Not Found."}, 404
+
+        db.session.query(EvaluationResult)\
+            .filter(EvaluationResult.evaluation_result_id == eval_result_id).delete()
+        db.session.commit()
+        db.session.close()
+
+        return {"status": True, "message": "Success."}

app/apis/api_service.py

@@ -1,11 +1,8 @@
 import datetime
-import json
 
 from flask_restplus import Namespace, fields, Resource, reqparse
 
-from app import logger
-from models import db, Kubernetes, Application, Service, EvaluationResult, Evaluation
-from core.drucker_dashboard_client import DruckerDashboardClient
+from models import db, Kubernetes, Application, Service
 
 from apis.common import DatetimeToTimestamp
 from apis.api_kubernetes import update_dbs_kubernetes, switch_drucker_service_model_assignment
@@ -186,54 +183,3 @@ def delete(self, application_id:int, service_id:int):
         db.session.commit()
         db.session.close()
         return response_body
-
-@srv_info_namespace.route('/<int:application_id>/services/<int:service_id>/evaluate')
-class ApiEvaluate(Resource):
-    eval_parser = reqparse.RequestParser()
-    eval_parser.add_argument('evaluation_id', location='form', type=int, required=False)
-    eval_parser.add_argument('overwrite', location='form', type=bool, required=False)
-
-    @srv_info_namespace.expect(eval_parser)
-    def post(self, application_id:int, service_id:int):
-        """evaluate"""
-        args = self.eval_parser.parse_args()
-        eval_id = args.get('evaluation_id', None)
-        if eval_id:
-            eobj = Evaluation.query.filter_by(
-                application_id=application_id,
-                evaluation_id=eval_id).first_or_404()
-        else:
-            # if evaluation_id is not given, use the lastest one.
-            eobj = Evaluation.query\
-                .filter_by(application_id=application_id)\
-                .order_by(Evaluation.register_date.desc()).first_or_404()
-
-        sobj = Service.query.filter_by(
-            application_id=application_id,
-            service_id=service_id).first_or_404()
-
-        robj = db.session.query(EvaluationResult)\
-            .filter(EvaluationResult.model_id == sobj.model_id,
-                    EvaluationResult.evaluation_id == eobj.evaluation_id).one_or_none()
-        if robj is not None and args.get('overwrite', False):
-            return json.loads(robj.result)
-
-        eval_result_path = "eval-result-{0:%Y%m%d%H%M%S}.txt".format(datetime.datetime.utcnow())
-        drucker_dashboard_application = DruckerDashboardClient(logger=logger, host=sobj.host)
-        response_body = drucker_dashboard_application.run_evaluate_model(eobj.data_path, eval_result_path)
-
-        if response_body['status']:
-            result = json.dumps(response_body)
-            if robj is None:
-                robj = EvaluationResult(model_id=sobj.model_id,
-                                        data_path=eval_result_path,
-                                        evaluation_id=eobj.evaluation_id,
-                                        result=result)
-                db.session.add(robj)
-            else:
-                robj.data_path = eval_result_path
-                robj.result = result
-            db.session.commit()
-            db.session.close()
-
-        return response_body

app/core/drucker_dashboard_client.py

@@ -138,3 +138,33 @@ def run_upload_evaluation_data(self, f:FileStorage, data_path:str):
         response = protobuf_to_dict(self.stub.UploadEvaluationData(request_iterator),
                                     including_default_value_fields=True)
         return response
+
+    def __get_value_from_io(self, io:drucker_pb2.IO):
+        if io.WhichOneof('io_oneof') == 'str':
+            val = io.str.val
+        else:
+            val = io.tensor.val
+
+        if len(val) == 1:
+            return val[0]
+        else:
+            return list(val)
+
+    @error_handling({"status": False})
+    def run_evaluation_data(self, data_path:str, result_path:str):
+        request = drucker_pb2.EvaluationResultRequest(data_path=data_path, result_path=result_path)
+        for raw_response in self.stub.EvaluationResult(request):
+            details = []
+            for detail in raw_response.detail:
+                details.append(dict(
+                    protobuf_to_dict(detail, including_default_value_fields=True),
+                    input=self.__get_value_from_io(detail.input),
+                    label=self.__get_value_from_io(detail.label),
+                    output=self.__get_value_from_io(detail.output),
+                    score=detail.score[0] if len(detail.score) == 1 else list(detail.score)
+                ))
+            response = protobuf_to_dict(raw_response,
+                                        including_default_value_fields=True)
+            response['detail'] = details
+            response['status'] = True
+            yield response