Changes made for Multiple Codegen (#8)

* Update __init__.py

* read variable from environ

* Update post_operation.py

* app config file removal in codegen - xAOD

* Update post_operation.py

* os environ changes

* directly read from env var

* Final changes for multiple codegen

* Remove unused imports

* added back config transformer image

* deleted .DS_Store

* revert to current_app.config at post_operation and DS Store gitignore

* Update post_operation.py

* Pick up science image from app config

* Code generators return language and science command

---------

Co-authored-by: Prajwal  Kiran Kumar <[email protected]>
Co-authored-by: Ben Galewsky <[email protected]>

Author: 3 people

.gitignore

@@ -128,4 +128,7 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-.idea/
+.idea/
+
+#DS Store Mac OS
+.DS_Store

servicex_codegen/__init__.py

@@ -58,11 +58,11 @@ def create_app(test_config=None, provided_translator=None):
     if test_config:
         app.config.from_mapping(test_config)
     else:
-        if 'APP_CONFIG_FILE' in os.environ:
-            app.config.from_envvar('APP_CONFIG_FILE')
         if 'CODEGEN_CONFIG_FILE' in os.environ:
             app.config.from_envvar('CODEGEN_CONFIG_FILE')
 
+        app.config['TRANSFORMER_SCIENCE_IMAGE'] = os.environ.get('TRANSFORMER_SCIENCE_IMAGE')
+
     with app.app_context():
         translator = provided_translator
 

servicex_codegen/post_operation.py

@@ -26,16 +26,17 @@
 #   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 #   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-
+import json
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 import os
+
 import zipfile
 from tempfile import TemporaryDirectory
 
-from flask import Response, current_app, request
+from flask import Response, request, current_app
 from flask_restful import Resource
 from requests_toolbelt import MultipartEncoder
 
@@ -84,6 +85,10 @@ def stream_generated_code(self, generated_code_result: GeneratedFileResult) -> b
 
     def post(self):
         try:
+            with open("transformer_capabilities.json") as capabilities_file:
+                capabilities = json.load(capabilities_file)
+                print("capable", capabilities['language'], capabilities['command'])
+
             with TemporaryDirectory() as tempdir:
                 body = request.get_json()
                 generated_code_result = self.code_generator.generate_code(
@@ -92,12 +97,14 @@ def post(self):
                 zip_data = self.stream_generated_code(generated_code_result)
                 # code gen transformer returns the default transformer image mentioned in
                 # the config file
-                transformer_image = current_app.config.get("TRANSFORMER_SCIENCE_IMAGE")
+                transformer_image = current_app.config['TRANSFORMER_SCIENCE_IMAGE']
 
                 # MultipartEncoder library takes multiple types of data fields and merge
                 # them into a multipart mime data type
                 m = MultipartEncoder(
                     fields={'transformer_image': transformer_image,
+                            'language': capabilities['language'],
+                            'command': capabilities['command'],
                             'zip_data': zip_data}
                 )
 

tests/test_post_operation.py

@@ -66,7 +66,8 @@ def test_post_good_query_with_params(self, mocker):
             )
 
             config = {
-                'TARGET_BACKEND': 'uproot'
+                'TARGET_BACKEND': 'uproot',
+                'TRANSFORMER_SCIENCE_IMAGE': "foo/bar:latest"
             }
             app = create_app(config, provided_translator=mock_ast_translator)
             client = app.test_client()
@@ -85,7 +86,7 @@ def test_post_good_query_with_params(self, mocker):
         decoder_parts = decoder.MultipartDecoder(response.data, content_type)
 
         transformer_image = str(decoder_parts.parts[0].content, 'utf-8')
-        zip_file = decoder_parts.parts[1].content
+        zip_file = decoder_parts.parts[3].content
 
         print("Transformer Image: ", transformer_image)
         print("Zip File: ", zip_file)
@@ -130,7 +131,7 @@ def test_post_good_query_without_params(self, mocker):
             decoder_parts = decoder.MultipartDecoder(response.data, content_type)
 
             transformer_image = str(decoder_parts.parts[0].content, 'utf-8')
-            zip_file = decoder_parts.parts[1].content
+            zip_file = decoder_parts.parts[3].content
 
             print("Transformer Image: ", transformer_image)
             print("Zip File: ", zip_file)

transformer_capabilities.json

@@ -0,0 +1,9 @@
+{
+  "name": "FuncADL based uproot transformer",
+  "description": "Extracts data from flat ntuple style root files.",
+  "limitations": "Would be good to note what isn't implemented",
+  "file-formats": ["parquet"],
+  "stats-parser": "UprootStats",
+  "language": "python",
+  "command": "/generated/transform_single_file.py"
+}