From 3c6dcb9db964214212c498406cd9337e416ba437 Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Fri, 25 Aug 2023 14:20:48 +0000 Subject: [PATCH 01/12] fix raise_exception --- nbtools/run_notebook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index 9734012..00ab026 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -310,9 +310,6 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p if outputs is not None: executor.kc = kernel_manager.client() # For compatibility with 5.x.x version of `nbconvert` executor.preprocess_cell(output_cell, {'metadata': {'path': working_dir}}, -1) - - if raise_exception: - raise finally: # Shutdown kernel kernel_manager.cleanup_resources() @@ -337,6 +334,10 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p for path_ in db_paths: os.remove(path_) + # Re-raise exception if needed + if failed and raise_exception: + raise Exception(traceback_message) + # Prepare execution results: execution state, notebook outputs and error info (if exists) if failed: exec_res = {'failed': failed, 'failed cell number': error_cell_num, 'traceback': traceback_message} From 24119d6f04d907a7414e447afe2b47a17618d7cc Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Fri, 25 Aug 2023 14:30:13 +0000 Subject: [PATCH 02/12] add proper exception raise for process --- nbtools/run_notebook.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index 00ab026..4307d5b 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -48,6 +48,9 @@ def _wrapper(*args, **kwargs): if json_path is not None and os.path.exists(json_path): os.remove(json_path) + if kwargs.get('raise_exception') and output['failed']: + raise Exception(output['traceback']) + return output return _wrapper @@ -334,13 +337,14 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p for path_ in db_paths: os.remove(path_) - # Re-raise exception if needed - if failed and raise_exception: - raise Exception(traceback_message) - # Prepare execution results: execution state, notebook outputs and error info (if exists) if failed: exec_res = {'failed': failed, 'failed cell number': error_cell_num, 'traceback': traceback_message} + + # Re-raise exception if needed + if raise_exception: + returned_value.put(exec_res) + return None else: exec_res = {'failed': failed, 'failed cell number': None, 'traceback': ''} From 1062d555705835f0d26f6f4404103023b68c93af Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Fri, 25 Aug 2023 14:31:54 +0000 Subject: [PATCH 03/12] add proper exception raise for process --- nbtools/run_notebook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index 4307d5b..048a256 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -48,7 +48,7 @@ def _wrapper(*args, **kwargs): if json_path is not None and os.path.exists(json_path): os.remove(json_path) - if kwargs.get('raise_exception') and output['failed']: + if kwargs.get('raise_exception', False) and output['failed']: raise Exception(output['traceback']) return output From d4688b9535fee0e2d0dde27a936894df46a13ebe Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Fri, 25 Aug 2023 14:41:22 +0000 Subject: [PATCH 04/12] fix pylint --- nbtools/run_notebook.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index 048a256..dd1b4ba 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -19,7 +19,7 @@ def run_in_process(func): """ Decorator to run the `func` in a separated process for terminating all relevant processes properly. """ @wraps(func) def _wrapper(*args, **kwargs): - # pylint: disable=bare-except + # pylint: broad-exception-raised returned_value = Queue() kwargs = {**kwargs, 'returned_value': returned_value} @@ -31,12 +31,15 @@ def _wrapper(*args, **kwargs): path = args[0] if args else kwargs['path'] json_path = f'{TMP_DIR}/{process.pid}.json' + with open(json_path, 'w', encoding='utf-8') as file: json.dump({'path': path}, file) output = returned_value.get() process.join() - except: + except Exception as e: + output = {'failed': True, 'traceback': e} + # Terminate all relevant processes when something went wrong, e.g. Keyboard Interrupt for child in psutil.Process(process.pid).children(): if psutil.pid_exists(child.pid): @@ -378,7 +381,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p exec_res['notebook'] = notebook returned_value.put(exec_res) # return for parent process - return + return None # Mask functions for database operations cells def mask_inputs_reading(notebook, pos): From 60c9c82d7687d436151731652f19110f321a6eaa Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Fri, 25 Aug 2023 14:53:02 +0000 Subject: [PATCH 05/12] fix pylint --- nbtools/run_notebook.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index dd1b4ba..3570a83 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -19,12 +19,14 @@ def run_in_process(func): """ Decorator to run the `func` in a separated process for terminating all relevant processes properly. """ @wraps(func) def _wrapper(*args, **kwargs): - # pylint: broad-exception-raised + # pylint: broad-exception-caught, W0719 returned_value = Queue() kwargs = {**kwargs, 'returned_value': returned_value} json_path = None + output = {'failed': True, 'traceback': ''} + try: process = Process(target=func, args=args, kwargs=kwargs) process.start() From 396525646a3e3e33a58e213440e236c811710123 Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Fri, 25 Aug 2023 14:57:14 +0000 Subject: [PATCH 06/12] fix pylint --- nbtools/run_notebook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index 3570a83..1121817 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -19,7 +19,7 @@ def run_in_process(func): """ Decorator to run the `func` in a separated process for terminating all relevant processes properly. """ @wraps(func) def _wrapper(*args, **kwargs): - # pylint: broad-exception-caught, W0719 + # pylint: broad-exception-caught, broad-exception-raised returned_value = Queue() kwargs = {**kwargs, 'returned_value': returned_value} From 129f19622ed961d8070c7e6025414368323ad7cf Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Tue, 14 Nov 2023 12:46:02 +0000 Subject: [PATCH 07/12] add execute_kwargs doc --- nbtools/run_notebook.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index 1121817..d84440b 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -198,6 +198,9 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p Note, that database exists only if inputs and/or outputs are provided. execute_kwargs : dict, optional Parameters of `:class:ExecutePreprocessor`. + For example, you can provide timeout, kernel_name, resources (such as metadata) + and other `nbclient.client.NotebookClient` arguments from :ref:`the NotebookClient doc page + `. add_timestamp : bool, optional Whether to add a cell with execution information at the beginning of the saved notebook. hide_code_cells : bool, optional From c4bb6c401297cfbf4d40d7e917e0f3982403889a Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Tue, 14 Nov 2023 13:58:15 +0000 Subject: [PATCH 08/12] fix pylint --- nbtools/run_notebook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index d84440b..ec68628 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -19,7 +19,7 @@ def run_in_process(func): """ Decorator to run the `func` in a separated process for terminating all relevant processes properly. """ @wraps(func) def _wrapper(*args, **kwargs): - # pylint: broad-exception-caught, broad-exception-raised + # pylint: disable=broad-exception-caught, broad-exception-raised returned_value = Queue() kwargs = {**kwargs, 'returned_value': returned_value} @@ -235,7 +235,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p Executed notebook object. Note that this output is provided only if `return_notebook` is True. """ - # pylint: disable=bare-except, lost-exception + # pylint: disable=bare-except, lost-exception, return-in-finally import nbformat from jupyter_client.manager import KernelManager from nbconvert.preprocessors import ExecutePreprocessor @@ -386,7 +386,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p exec_res['notebook'] = notebook returned_value.put(exec_res) # return for parent process - return None + return None # Mask functions for database operations cells def mask_inputs_reading(notebook, pos): From 789df3a1423b5e8835c462dc2016c18b7868d514 Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Tue, 5 Dec 2023 16:48:54 +0000 Subject: [PATCH 09/12] fix pylint_notebook and update to pylint 3.0.2 --- nbtools/pylint_notebook.py | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/nbtools/pylint_notebook.py b/nbtools/pylint_notebook.py index dc7ff20..2fb9a10 100755 --- a/nbtools/pylint_notebook.py +++ b/nbtools/pylint_notebook.py @@ -1,6 +1,7 @@ """ Functions for code quality control of Jupyter Notebooks. """ #pylint: disable=import-outside-toplevel import os +import sys from .core import StringWithDisabledRepr, get_notebook_path, notebook_to_script @@ -101,7 +102,10 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, pylint_params : dict Additional parameter of linting. Each is converted to a separate valid entry in the `pylintrc` file. """ - from pylint import epylint as lint + from io import StringIO + from pylint.lint import Run + from pylint.reporters.text import TextReporter + path = path or get_notebook_path() if path is None: raise ValueError('Provide path to Jupyter Notebook or run `pylint_notebook` inside of it!') @@ -117,12 +121,24 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, pylintrc = generate_pylintrc(path_pylintrc, disable=disable, enable=enable, **pylint_params) # Run pylint on script with pylintrc configuration - pylint_cmdline = ' '.join([path_pylintrc, f'--rcfile {path_pylintrc}', *options]) - pylint_stdout, pylint_stderr = lint.py_run(pylint_cmdline, return_std=True) + pylint_cmdline = [path_pylintrc, f'--rcfile={path_pylintrc}', *options] + + # Custom streams for catching messagies + pylint_stdout = StringIO() + + base_stderr = sys.stderr + sys.stderr = StringIO() - errors = pylint_stderr.getvalue() + # Run pylint + reporter = TextReporter(pylint_stdout) + Run(pylint_cmdline, reporter=reporter, exit=False) + + # Parse outputs and get back stderr report = pylint_stdout.getvalue() + errors = sys.stderr.getvalue() + sys.stderr = base_stderr + # Prepare custom report output = [] @@ -133,9 +149,10 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, elif path_script in line or script_name in line: line = line.replace(path_script, '__').replace(script_name, '__') + line_semicolon_split = line.split(':') # Locate the cell and line inside the cell - code_line_number = int(line.split(':')[1]) + code_line_number = int(line_semicolon_split[1]) for cell_number, cell_ranges in cell_line_numbers.items(): if code_line_number in cell_ranges: cell_line_number = code_line_number - cell_ranges[0] @@ -144,12 +161,12 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, cell_number, cell_line_number = -1, code_line_number # Find error_code and error_name: for example, `C0123` and `invalid-name` - position_left = line.find('(') + 1 - position_right = line.find(')') - 1 - error_message = line[position_left : position_right] - error_human_message = line[position_right + 2:] - error_code, error_name, *_ = error_message.split(',') - error_name = error_name.strip() + error_code = line_semicolon_split[3].strip() + + error_message = line_semicolon_split[4].strip() + error_human_message, error_name = error_message.split('(') + error_human_message = error_human_message.strip() + error_name = error_name[:-1] # Make new message message = f'Cell {cell_number}:{cell_line_number}, code={error_code}, name={error_name}' From d67732162b0e02b39e517a9e9bb6ec5563c93f91 Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Tue, 5 Dec 2023 17:07:49 +0000 Subject: [PATCH 10/12] add double brackets error --- nbtools/pylint_notebook.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nbtools/pylint_notebook.py b/nbtools/pylint_notebook.py index 2fb9a10..323b907 100755 --- a/nbtools/pylint_notebook.py +++ b/nbtools/pylint_notebook.py @@ -143,6 +143,8 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, output = [] for line in report.split('\n'): + # error line is a str in the format: + # "filename.py:line_num:position_num: error_code: error_text (error_name)" if 'rated' in line: output.insert(0, line.strip(' ')) output.insert(1, '–' * (len(line) - 1)) @@ -164,9 +166,10 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, error_code = line_semicolon_split[3].strip() error_message = line_semicolon_split[4].strip() - error_human_message, error_name = error_message.split('(') - error_human_message = error_human_message.strip() - error_name = error_name[:-1] + error_name_start = error_message.rfind('(') + + error_human_message = error_message[:error_name_start].strip() + error_name = error_message[error_name_start+1:-1] # Make new message message = f'Cell {cell_number}:{cell_line_number}, code={error_code}, name={error_name}' From 3deb728f779a9fb435b5e76a0c2a941d4a0f6daa Mon Sep 17 00:00:00 2001 From: HollowPrincess Date: Wed, 20 Dec 2023 14:32:08 +0000 Subject: [PATCH 11/12] PR comments fixes: little prettifyings --- nbtools/pylint_notebook.py | 32 ++++++++++++++------------------ nbtools/run_notebook.py | 14 +++++++------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/nbtools/pylint_notebook.py b/nbtools/pylint_notebook.py index 323b907..9fc35d4 100755 --- a/nbtools/pylint_notebook.py +++ b/nbtools/pylint_notebook.py @@ -1,7 +1,8 @@ """ Functions for code quality control of Jupyter Notebooks. """ #pylint: disable=import-outside-toplevel import os -import sys +from io import StringIO +from contextlib import redirect_stderr, redirect_stdout from .core import StringWithDisabledRepr, get_notebook_path, notebook_to_script @@ -102,9 +103,12 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, pylint_params : dict Additional parameter of linting. Each is converted to a separate valid entry in the `pylintrc` file. """ - from io import StringIO - from pylint.lint import Run - from pylint.reporters.text import TextReporter + try: + from pylint.lint import Run + from pylint.reporters.text import TextReporter + except ImportError as exception: + raise ImportError('Install pylint') from exception + path = path or get_notebook_path() if path is None: @@ -123,21 +127,13 @@ def pylint_notebook(path=None, options=(), disable=(), enable=(), printer=print, # Run pylint on script with pylintrc configuration pylint_cmdline = [path_pylintrc, f'--rcfile={path_pylintrc}', *options] - # Custom streams for catching messagies - pylint_stdout = StringIO() - - base_stderr = sys.stderr - sys.stderr = StringIO() - - # Run pylint - reporter = TextReporter(pylint_stdout) - Run(pylint_cmdline, reporter=reporter, exit=False) - - # Parse outputs and get back stderr - report = pylint_stdout.getvalue() + # Run pylint and catch messagies + with redirect_stdout(StringIO()) as pylint_stdout, redirect_stderr(StringIO()) as pylint_stderr: + reporter = TextReporter(pylint_stdout) + Run(pylint_cmdline, reporter=reporter, exit=False) - errors = sys.stderr.getvalue() - sys.stderr = base_stderr + report = pylint_stdout.getvalue() + errors = pylint_stderr.getvalue() # Prepare custom report output = [] diff --git a/nbtools/run_notebook.py b/nbtools/run_notebook.py index ec68628..e59c524 100755 --- a/nbtools/run_notebook.py +++ b/nbtools/run_notebook.py @@ -20,8 +20,8 @@ def run_in_process(func): @wraps(func) def _wrapper(*args, **kwargs): # pylint: disable=broad-exception-caught, broad-exception-raised - returned_value = Queue() - kwargs = {**kwargs, 'returned_value': returned_value} + _output_queue = Queue() + kwargs = {**kwargs, '_output_queue': _output_queue} json_path = None @@ -37,7 +37,7 @@ def _wrapper(*args, **kwargs): with open(json_path, 'w', encoding='utf-8') as file: json.dump({'path': path}, file) - output = returned_value.get() + output = _output_queue.get() process.join() except Exception as e: output = {'failed': True, 'traceback': e} @@ -138,7 +138,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p working_dir = './', execute_kwargs=None, out_path_db=None, out_path_ipynb=None, out_path_html=None, remove_db='always', add_timestamp=True, hide_code_cells=False, mask_extra_code=False, display_links=True, - raise_exception=False, return_notebook=False, returned_value=None): + raise_exception=False, return_notebook=False, _output_queue=None): """ Execute a Jupyter Notebook programmatically. Heavily inspired by https://github.com/tritemio/nbrun. @@ -214,7 +214,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p Whether to re-raise exceptions from the notebook. return_notebook : bool, optional Whether to return the notebook object from this function. - returned_value : None + _output_queue : None Placeholder for the :func:`~.run_in_process` decorator to return this function result. Returns @@ -351,7 +351,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p # Re-raise exception if needed if raise_exception: - returned_value.put(exec_res) + _output_queue.put(exec_res) return None else: exec_res = {'failed': failed, 'failed cell number': None, 'traceback': ''} @@ -385,7 +385,7 @@ def run_notebook(path, inputs=None, outputs=None, inputs_pos=1, replace_inputs_p if return_notebook: exec_res['notebook'] = notebook - returned_value.put(exec_res) # return for parent process + _output_queue.put(exec_res) # return for parent process return None # Mask functions for database operations cells From 1e41add3e93eacda5c43cc6ef0da04383808c53f Mon Sep 17 00:00:00 2001 From: SergeyTsimfer <47103382+SergeyTsimfer@users.noreply.github.com> Date: Fri, 22 Dec 2023 13:19:53 +0300 Subject: [PATCH 12/12] Bump version to 0.9.12 --- nbtools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbtools/__init__.py b/nbtools/__init__.py index f7b4c44..310a83e 100755 --- a/nbtools/__init__.py +++ b/nbtools/__init__.py @@ -4,4 +4,4 @@ from .run_notebook import run_notebook from .pylint_notebook import pylint_notebook -__version__ = '0.9.11' +__version__ = '0.9.12'