deploy: 2a9df7c

analysiscenter · Jul 22, 2024 · ab41891 · ab41891
commit ab41891
Show file tree

Hide file tree

Showing 67 changed files with 9,436 additions and 0 deletions.
diff --git a/.buildinfo b/.buildinfo
@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 747acfc6cc0b44b110f1f84d05a3f644
+tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/.doctrees/api/modules.doctree b/.doctrees/api/modules.doctree
diff --git a/.doctrees/api/nbtools.doctree b/.doctrees/api/nbtools.doctree
diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle
diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree
diff --git a/.doctrees/user_guide/nbtools.doctree b/.doctrees/user_guide/nbtools.doctree
diff --git a/.doctrees/user_guide/nbtools.library.doctree b/.doctrees/user_guide/nbtools.library.doctree
diff --git a/.doctrees/user_guide/nbtools.nbstat.doctree b/.doctrees/user_guide/nbtools.nbstat.doctree
diff --git a/.nojekyll b/.nojekyll
diff --git a/_images/nbwatch.gif b/_images/nbwatch.gif
diff --git a/_modules/index.html b/_modules/index.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Overview: module code &mdash; nbtools  documentation</title>
+      <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
+      <link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
+
+
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+
+        <script src="../_static/jquery.js?v=5d32c60e"></script>
+        <script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+        <script src="../_static/documentation_options.js?v=5929fcd5"></script>
+        <script src="../_static/doctools.js?v=9a2dae69"></script>
+        <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+
+
+          <a href="../index.html">
+
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../user_guide/nbtools.html">Getting Started</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../api/modules.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">nbtools</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
+      <li class="breadcrumb-item active">Overview: module code</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+
+  <h1>All modules for which code is available</h1>
+<ul><li><a href="nbtools/core.html">nbtools.core</a></li>
+<li><a href="nbtools/exec_notebook.html">nbtools.exec_notebook</a></li>
+<li><a href="nbtools/nbstat/resource_inspector.html">nbtools.nbstat.resource_inspector</a></li>
+<li><a href="nbtools/pylint_notebook.html">nbtools.pylint_notebook</a></li>
+</ul>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2024, Analysis Center.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
diff --git a/_modules/nbtools/core.html b/_modules/nbtools/core.html
diff --git a/_modules/nbtools/exec_notebook.html b/_modules/nbtools/exec_notebook.html
diff --git a/_modules/nbtools/nbstat/resource_inspector.html b/_modules/nbtools/nbstat/resource_inspector.html
diff --git a/_modules/nbtools/pylint_notebook.html b/_modules/nbtools/pylint_notebook.html
diff --git a/_sources/api/modules.rst.txt b/_sources/api/modules.rst.txt
@@ -0,0 +1,8 @@
+=============
+API reference
+=============
+
+.. toctree::
+   :maxdepth: 3
+
+   nbtools
diff --git a/_sources/api/nbtools.rst.txt b/_sources/api/nbtools.rst.txt
@@ -0,0 +1,22 @@
+nbtools
+=============
+
+Linting and execution
+****************************************
+
+.. automodule:: nbtools.exec_notebook
+   :members: exec_notebook, extract_traceback, run_in_process
+
+.. automodule:: nbtools.pylint_notebook
+   :members: pylint_notebook
+
+GPU utils
+*********
+
+.. automodule:: nbtools.core
+   :members: free_gpus, get_available_gpus, get_gpu_free_memory, set_gpus
+
+Monitoring tools
+****************
+
+.. autoclass:: nbtools.nbstat.resource_inspector.ResourceInspector
diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt
@@ -0,0 +1,48 @@
+Welcome to NBtools
+==================
+
+Collection of tools for monitoring running Jupyter Notebooks and for their execition and linting.
+
+The main tool of this package is **nbstat** / **nbwatch** command line utility. It is added at installation and shows the detailed resource utilization for each process of each running Jupyter Notebook. A gif is worth a thousand words:
+
+.. image:: ../images/nbwatch.gif
+    :width: 90%
+
+
+Installation
+************
+
+The installation should be as easy as:
+
+.. code-block:: bash
+
+   pip install py-nbtools
+
+Get started
+***********
+
+.. toctree::
+   :maxdepth: 2
+
+   user_guide/nbtools
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+
+   api/modules
+
+Goals
+*****
+
+This library started as a container of tools, that I came across / developed in my years as an ML researcher. As some of the functions survived multiple refactoring iterations, I decided to share the library so it is easier to perfect them and test in different environments.
+
+Another goal of the project is to show how to communicate with Jupyter API on real world examples: instead of reading through a number of stackoverflow threads, you can find the same information collected in one place and get a rough understanding of what is possible with it and what is not.
+
+Acknowledgements
+****************
+
+The **nbstat** module builds on `gpustat <https://github.com/wookayin/gpustat/>`_ package. Using the **gpustat** for years gave me an idea about possible improvements, which are implemented in this library. While the implementation is different, reading through the code of **gpustat** was essential for development.
+
+Animated GIFs are created by using `Terminalizer <https://github.com/faressoft/terminalizer>`_: aside from the usual problems with installation, the tool itself is amazing.
+
diff --git a/_sources/user_guide/nbtools.library.rst.txt b/_sources/user_guide/nbtools.library.rst.txt
@@ -0,0 +1,65 @@
+========================================
+Jupyter Notebooks: linting and execution
+========================================
+
+Other than `nbstat / nbwatch` monitoring utilities, this library provides a few useful tools for working with notebooks and GPUs.
+
+
+pylint_notebook
+***************
+
+Shamelessly taken from `pylint page <https://pylint.pycqa.org/en/latest/>`_
+
+Function that checks for errors in Jupyter Notebooks with Python code, tries to enforce a coding standard and looks for code smells. It can also look for certain type errors, it can recommend suggestions about how particular blocks can be refactored and can offer you details about the code's complexity.
+
+Using it as easy as:
+
+.. code-block:: python
+
+    from nbtools import pylint_notebook
+    pylint_notebook(path_to_ipynb,             # If not provided, use path to the current notebook
+                    disable='invalid-name',    # Disable specified Pylint checks. Can be a list.
+                    enable='import-error')     # Enable  specified Pylint checks. Can be a list.
+
+
+Under the hood, it converts `.ipynb` notebook to `.py` script, creates a custom `.pylintrc` configuration, runs the `pylint` and removes all temporary files. Learn more about its usage in the [tutorial.](tutorials/NBstat.ipynb)
+
+exec_notebook
+*************
+
+Provides a `eval`-like interface for running Jupyter Notebooks programmatically. We use it for running interactive tests, that are easy to work with: in case of any failures, one can jump right into fixing it with an already set-up environment.
+
+.. code-block:: python
+
+    from nbtools import exec_notebook
+    exec_notebook(path_to_ipynb,                       # Which notebook to run
+                  out_path_ipynb,                      # Where to save result
+                  inputs={'learning_rate': 0.05,},     # Pass variables to notebook
+                  outputs=['accuracy'])                # Extract variables from notebook
+
+
+set_gpus, free_gpus
+*******************
+
+Select free device(s) and set `CUDA_VISIBLE_DEVICES` environment variable so that the current process sees only them.
+
+Eliminates an enormous amount of bugs and unexpected behaviors related to GPU usage.
+
+.. code-block:: python
+
+    from nbtools import set_gpus, free_gpus
+    used_gpus = set_gpus(n=2,                 # Number of devices to set.
+                         min_free_memory=0.7, # Minimum amount of free memory on device to consider free.
+                         max_processes=3)     # Maximum amount of  processes  on device to consider free.
+    free_gpus(used_gpus)                      # Kill all processes on selected GPUs. Useful at teardown.
+
+Other functions
+***************
+
+.. code-block:: python
+
+    from nbtools import (in_notebook,         # Return True if executed inside of Jupyter Notebook
+                         get_notebook_path,   # If executed in Jupyter Notebook, return its absolute path
+                         get_notebook_name,   # If executed in Jupyter Notebook, return its name
+                         notebook_to_script)  # Convert Jupyter Notebook to an executable Python script.
+                                              # Works well with magics and command line executions.