Refactor to use CodeGen library

requirements.txt

@@ -1,14 +1,4 @@
 # Requirements needed to run the app.
-Flask==1.1.2
-Flask-RESTful==0.3.8
-Flask-WTF==0.14.3
+servicex-code-gen-lib==0.1a9
 func-adl-xAOD==2.0.1
 
-# pinned since Flask doesn't work with latest version of itsdangerous
-itsdangerous==2.0.1
-# Pinned back to the bug-fix releases of 2.0 as that matches
-# where flask 1.1.2 was released
-jinja2==2.11.3
-# And jinja2 requires a less recent version of markupsafe and Werkzeug
-markupsafe==1.1.1
-Werkzeug==1.0.1

servicex/code_generator_service/__init__.py

@@ -25,48 +25,20 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import os
+from servicex.code_generator_service.ast_translator import AstAODTranslator
 
-from flask import Flask
-from flask_restful import Api
-from servicex.code_generator_service.generate_code import GenerateCode
-from servicex.code_generator_service.ast_translator import AstTranslator
-
-
-def handle_invalid_usage(error: BaseException):
-    from flask import jsonify
-    response = jsonify({"message": str(error)})
-    response.status_code = 400
-    return response
+import servicex_codegen
+from flask.config import Config
 
 
 def create_app(test_config=None, provided_translator=None):
-    """Create and configure an instance of the Flask application."""
-    app = Flask(__name__, instance_relative_config=True)
-
-    # ensure the instance folder exists
-    try:
-        os.makedirs(app.instance_path)
-    except OSError:
-        pass
-
-    if not test_config:
-        app.config.from_envvar('APP_CONFIG_FILE')
-    else:
-        app.config.from_mapping(test_config)
-
-    with app.app_context():
-
-        if not provided_translator:
-            translator = AstTranslator(app.config['TARGET_BACKEND'])
-        else:
-            translator = provided_translator
-
-        api = Api(app)
-        GenerateCode.make_api(translator)
-
-        api.add_resource(GenerateCode, '/servicex/generated-code')
-
-    app.errorhandler(Exception)(handle_invalid_usage)
-
-    return app
+    # We need access to the App's config to determine translater backend before we create
+    # the app, so drive the flask config machinery directly
+    app_config = Config(".")
+    app_config.from_envvar("APP_CONFIG_FILE")
+
+    return servicex_codegen.create_app(test_config,
+                                       provided_translator=provided_translator
+                                       if provided_translator else
+                                       AstAODTranslator(app_config['TARGET_BACKEND'])
+                                       )

servicex/code_generator_service/ast_translator.py

@@ -25,30 +25,20 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import ast
 import os
-import zipfile
-from collections import namedtuple
-from tempfile import TemporaryDirectory
-from typing import Optional, Union
 from pathlib import Path
+from typing import Optional, Union
 
 from func_adl_xAOD.atlas.xaod.executor import atlas_xaod_executor
 from func_adl_xAOD.cms.aod.executor import cms_aod_executor
 from func_adl_xAOD.common.executor import executor
 from qastle import text_ast_to_python_ast
 
-GeneratedFileResult = namedtuple('GeneratedFileResult', 'hash output_dir')
-
-
-class GenerateCodeException(BaseException):
-    """Custom exception for top level code generation exceptions"""
+from servicex_codegen.code_generator import CodeGenerator, GeneratedFileResult, \
+    GenerateCodeException
 
-    def __init__(self, message: str):
-        BaseException.__init__(self, message)
 
-
-class AstTranslator:
+class AstAODTranslator(CodeGenerator):
     def __init__(self, executor: Optional[Union[executor, str]] = None):
         '''
         Create the ast translator objects
@@ -64,63 +54,32 @@ def __init__(self, executor: Optional[Union[executor, str]] = None):
             elif executor == 'ATLAS xAOD':
                 self._exe = atlas_xaod_executor()
             else:
-                raise ValueError(f'The executor name, {executor}, must be "CMS AOD" or "ATLAS xAOD" only.')
+                raise ValueError(f'The executor name, {executor}, must be "CMS AOD" or "ATLAS xAOD" only.')  # noqa: E501
         else:
             self._exe = executor
 
     @property
     def executor(self):
         return self._exe
 
-    def _zipdir(self, dir: Path, zip_handle: zipfile.ZipFile) -> None:
-        """Given a `path` to a directory, zip up its contents into a zip file.
-
-        Arguments:
-            path        Path to a local directory. The contents will be put into the zip file
-            zip_handle  The zip file handle to write into.
-        """
-        for root, _, files in os.walk(dir):
-            for file in files:
-                zip_handle.write(os.path.join(root, file), file)
-
-    def get_generated_xAOD(self, a: ast.AST, query_dir: Path):
-        if not query_dir.exists():
-            query_dir.mkdir(parents=True, exist_ok=True)
+    def generate_code(self, query, cache_path: str):
+        path = Path(cache_path)
+        if not path.exists():
+            path.mkdir(parents=True, exist_ok=True)
 
-        self._exe.write_cpp_files(
-            self._exe.apply_ast_transformations(a), query_dir)
-
-    def translate_text_ast_to_zip(self, code: str) -> bytes:
-        """Translate a text ast into a zip file as a memory stream
-
-        Arguments:
-            code            Text `qastle` version of the input ast generated by func_adl
-
-        Returns
-            bytes       Data that if written as a binary output would be a zip file.
-        """
-
-        if len(code) == 0:
+        if len(query) == 0:
             raise GenerateCodeException("Requested codegen for an empty string.")
 
-        body = text_ast_to_python_ast(code).body
-        print("------>", code, body)
+        body = text_ast_to_python_ast(query).body
+        print("------>", query, body)
         if len(body) != 1:
             raise GenerateCodeException(
-                f'Requested codegen for "{code}" yielded no code statements (or too many).')  # noqa: E501
+                f'Requested codegen for "{query}" yielded no code statements (or too many).')  # noqa: E501
         a = body[0].value
 
-        # Generate the C++ code
-        with TemporaryDirectory() as tempdir:
-            loc = Path(tempdir) / 'hash'
-            self.get_generated_xAOD(a, loc)
+        self._exe.write_cpp_files(
+            self._exe.apply_ast_transformations(a), path)
 
-            # Zip up everything in the directory - we are going to ship it as back as part
-            # of the message.
-            z_filename = Path(tempdir) / 'joined.zip'
-            zip_h = zipfile.ZipFile(z_filename, 'w', zipfile.ZIP_DEFLATED)
-            self._zipdir(loc, zip_h)
-            zip_h.close()
+        os.system("ls -lht " + cache_path)
 
-            with z_filename.open('rb') as b_in:
-                return b_in.read()
+        return GeneratedFileResult(hash, cache_path)