diff --git a/requirements.txt b/requirements.txt index 8bc3f26..31d774d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,4 @@ # Requirements needed to run the app. -Flask==1.1.2 -Flask-RESTful==0.3.8 -Flask-WTF==0.14.3 +servicex-code-gen-lib==0.1a9 func-adl-xAOD==2.0.1 -# pinned since Flask doesn't work with latest version of itsdangerous -itsdangerous==2.0.1 -# Pinned back to the bug-fix releases of 2.0 as that matches -# where flask 1.1.2 was released -jinja2==2.11.3 -# And jinja2 requires a less recent version of markupsafe and Werkzeug -markupsafe==1.1.1 -Werkzeug==1.0.1 \ No newline at end of file diff --git a/servicex/code_generator_service/__init__.py b/servicex/code_generator_service/__init__.py index 30a7386..0d1d157 100644 --- a/servicex/code_generator_service/__init__.py +++ b/servicex/code_generator_service/__init__.py @@ -25,48 +25,20 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os +from servicex.code_generator_service.ast_translator import AstAODTranslator -from flask import Flask -from flask_restful import Api -from servicex.code_generator_service.generate_code import GenerateCode -from servicex.code_generator_service.ast_translator import AstTranslator - - -def handle_invalid_usage(error: BaseException): - from flask import jsonify - response = jsonify({"message": str(error)}) - response.status_code = 400 - return response +import servicex_codegen +from flask.config import Config def create_app(test_config=None, provided_translator=None): - """Create and configure an instance of the Flask application.""" - app = Flask(__name__, instance_relative_config=True) - - # ensure the instance folder exists - try: - os.makedirs(app.instance_path) - except OSError: - pass - - if not test_config: - app.config.from_envvar('APP_CONFIG_FILE') - else: - app.config.from_mapping(test_config) - - with app.app_context(): - - if not provided_translator: - translator = AstTranslator(app.config['TARGET_BACKEND']) - else: - translator = provided_translator - - api = Api(app) - GenerateCode.make_api(translator) - - api.add_resource(GenerateCode, '/servicex/generated-code') - - app.errorhandler(Exception)(handle_invalid_usage) - - return app + # We need access to the App's config to determine translater backend before we create + # the app, so drive the flask config machinery directly + app_config = Config(".") + app_config.from_envvar("APP_CONFIG_FILE") + + return servicex_codegen.create_app(test_config, + provided_translator=provided_translator + if provided_translator else + AstAODTranslator(app_config['TARGET_BACKEND']) + ) diff --git a/servicex/code_generator_service/ast_translator.py b/servicex/code_generator_service/ast_translator.py index 3f87872..0b61b97 100644 --- a/servicex/code_generator_service/ast_translator.py +++ b/servicex/code_generator_service/ast_translator.py @@ -25,30 +25,20 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import ast import os -import zipfile -from collections import namedtuple -from tempfile import TemporaryDirectory -from typing import Optional, Union from pathlib import Path +from typing import Optional, Union from func_adl_xAOD.atlas.xaod.executor import atlas_xaod_executor from func_adl_xAOD.cms.aod.executor import cms_aod_executor from func_adl_xAOD.common.executor import executor from qastle import text_ast_to_python_ast -GeneratedFileResult = namedtuple('GeneratedFileResult', 'hash output_dir') - - -class GenerateCodeException(BaseException): - """Custom exception for top level code generation exceptions""" +from servicex_codegen.code_generator import CodeGenerator, GeneratedFileResult, \ + GenerateCodeException - def __init__(self, message: str): - BaseException.__init__(self, message) - -class AstTranslator: +class AstAODTranslator(CodeGenerator): def __init__(self, executor: Optional[Union[executor, str]] = None): ''' Create the ast translator objects @@ -64,7 +54,7 @@ def __init__(self, executor: Optional[Union[executor, str]] = None): elif executor == 'ATLAS xAOD': self._exe = atlas_xaod_executor() else: - raise ValueError(f'The executor name, {executor}, must be "CMS AOD" or "ATLAS xAOD" only.') + raise ValueError(f'The executor name, {executor}, must be "CMS AOD" or "ATLAS xAOD" only.') # noqa: E501 else: self._exe = executor @@ -72,55 +62,24 @@ def __init__(self, executor: Optional[Union[executor, str]] = None): def executor(self): return self._exe - def _zipdir(self, dir: Path, zip_handle: zipfile.ZipFile) -> None: - """Given a `path` to a directory, zip up its contents into a zip file. - - Arguments: - path Path to a local directory. The contents will be put into the zip file - zip_handle The zip file handle to write into. - """ - for root, _, files in os.walk(dir): - for file in files: - zip_handle.write(os.path.join(root, file), file) - - def get_generated_xAOD(self, a: ast.AST, query_dir: Path): - if not query_dir.exists(): - query_dir.mkdir(parents=True, exist_ok=True) + def generate_code(self, query, cache_path: str): + path = Path(cache_path) + if not path.exists(): + path.mkdir(parents=True, exist_ok=True) - self._exe.write_cpp_files( - self._exe.apply_ast_transformations(a), query_dir) - - def translate_text_ast_to_zip(self, code: str) -> bytes: - """Translate a text ast into a zip file as a memory stream - - Arguments: - code Text `qastle` version of the input ast generated by func_adl - - Returns - bytes Data that if written as a binary output would be a zip file. - """ - - if len(code) == 0: + if len(query) == 0: raise GenerateCodeException("Requested codegen for an empty string.") - body = text_ast_to_python_ast(code).body - print("------>", code, body) + body = text_ast_to_python_ast(query).body + print("------>", query, body) if len(body) != 1: raise GenerateCodeException( - f'Requested codegen for "{code}" yielded no code statements (or too many).') # noqa: E501 + f'Requested codegen for "{query}" yielded no code statements (or too many).') # noqa: E501 a = body[0].value - # Generate the C++ code - with TemporaryDirectory() as tempdir: - loc = Path(tempdir) / 'hash' - self.get_generated_xAOD(a, loc) + self._exe.write_cpp_files( + self._exe.apply_ast_transformations(a), path) - # Zip up everything in the directory - we are going to ship it as back as part - # of the message. - z_filename = Path(tempdir) / 'joined.zip' - zip_h = zipfile.ZipFile(z_filename, 'w', zipfile.ZIP_DEFLATED) - self._zipdir(loc, zip_h) - zip_h.close() + os.system("ls -lht " + cache_path) - with z_filename.open('rb') as b_in: - return b_in.read() + return GeneratedFileResult(hash, cache_path) diff --git a/servicex/code_generator_service/generate_code.py b/servicex/code_generator_service/generate_code.py deleted file mode 100644 index 569c7bc..0000000 --- a/servicex/code_generator_service/generate_code.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2019, IRIS-HEP -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from flask import request, Response -from flask_restful import Resource -from servicex.code_generator_service.ast_translator import AstTranslator - - -class GenerateCode(Resource): - @classmethod - def make_api(cls, translator: AstTranslator): - cls.translator = translator - return cls - - def post(self): - try: - code = request.data.decode('utf8') - zip_data = self.translator.translate_text_ast_to_zip(code) - - # Send the response back to you-know-what. - response = Response( - response=zip_data, - status=200, mimetype='application/octet-stream') - return response - except BaseException as e: - print(str(e)) - import traceback - import sys - traceback.print_exc(file=sys.stdout) - return {'Message': str(e)}, 500 diff --git a/tests/test_ast_translator.py b/tests/test_ast_translator.py deleted file mode 100644 index 6d145aa..0000000 --- a/tests/test_ast_translator.py +++ /dev/null @@ -1,39 +0,0 @@ -from pathlib import Path - -import pytest -from func_adl_xAOD.backend.xAODlib.atlas_xaod_executor import \ - atlas_xaod_executor -from servicex.code_generator_service import AstTranslator -from servicex.code_generator_service.ast_translator import \ - GenerateCodeException - - -def test_ctor(): - 'Make sure default ctor works' - a = AstTranslator() - assert isinstance(a.executor, atlas_xaod_executor) - - -def test_translate_good(mocker): - exe = mocker.MagicMock() - - def write_files(a, p: str): - with (Path(p) / 'junk.txt').open('w') as b_out: - b_out.write("hi") - - exe.write_cpp_files.side_effect = write_files - - a = AstTranslator(xaod_executor=exe) - a.translate_text_ast_to_zip( - "(call ResultTTree (call Select (call SelectMany (call EventDataset (list 'localds://did_01')))))") - - exe.apply_ast_transformations.assert_called_once() - exe.write_cpp_files.assert_called_once() - - -def test_translate_no_code(mocker): - exe = mocker.MagicMock() - - a = AstTranslator(xaod_executor=exe) - with pytest.raises(GenerateCodeException): - a.translate_text_ast_to_zip("")