Merge branch 'develop' into 'main'

Release 0.2.0

See merge request hpi-xnor/bmxnet-projects/bitorch!43

.gitignore

@@ -154,3 +154,10 @@ tblogs/
 # other files
 *.csv
 
+# lightning checkpoints
+*.ckpt
+
+docs/source/_autosummary
+
+# wandb output files
+wandb

.gitlab-ci.yml

@@ -1,45 +1,42 @@
-image: python:3.8
+image: python:3.7
 
 before_script:
-    - python --version
-    - pip install --find-links https://download.pytorch.org/whl/torch_stable.html "torch==1.9.0+cu111" "torchvision==0.10.0+cu111"
-    - pip install -e .[dev]
-    - flake8 --version
-    - mypy --version
-    - pytest --version
+  - python --version
+  - pip install -e .[dev] --extra-index-url https://download.pytorch.org/whl/cu113
+  - pwd
+  - ls -l
+  - python -c "import sys;print(sys.path)"
 
-test-codestyle:3.7:
+.codestyle:
   stage: test
-  image: python:3.7
   script:
-    - pwd
-    - ls -l
-    - python -c "import sys;print(sys.path)"
-    - flake8 --config=setup.cfg
+    - flake8 --version
+    - mypy --version
+    - flake8
     - mypy --config-file mypy.ini
 
-test-units:3.7:
+codestyle:
+  extends: .codestyle
+
+codestyle:3.8:
+  extends: .codestyle
+  image: python:3.8
+
+.test:
   stage: test
-  image: python:3.7
   script:
-    - pwd
-    - ls -l
-    - python -c "import sys;print(sys.path)"
+    - pytest --version
     - python -m pytest .
 
-test-codestyle:
-  stage: test
-  script:
-    - pwd
-    - ls -l
-    - python -c "import sys;print(sys.path)"
-    - flake8 --config=setup.cfg
-    - mypy --config-file mypy.ini
+test:
+  extends: .test
+
+test:3.8:
+  extends: .test
+  image: python:3.8
 
-test-units:
+test-build-doc:
   stage: test
   script:
-    - pwd
-    - ls -l
-    - python -c "import sys;print(sys.path)"
-    - python -m pytest .
+    - apt-get update && apt-get install -y pandoc
+    - sphinx-build -b html docs/source/ docs/build/ -a

CHANGELOG.md

@@ -1,9 +1,36 @@
 # Changelog
 
 All notable changes to this project will be documented in this file.
-The format is based on [Keep a Changelog](http://keepachangelog.com/) 
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.2.0] - 2022/05/19
+
+### Added
+
+- automatic documentation generation using sphinx
+- more documentation of layers and modules
+- bit-width of quantization functions is now stored
+- new layers:
+  - [Pact](https://arxiv.org/abs/1805.06085) activation function
+  - QEmbedding
+  - QEmbeddingBag
+- fvbitcore support in the example scripts for flop and model size estimation on operation level
+
+### Changed
+
+- image classification example:
+  - script now uses [pytorch lightning](https://www.pytorchlightning.ai/)
+  - it includes distributed training capability
+  - added wandb metric logging
+- QConv layers can now be pickled
+- Different quantized versions of LeNet available
+
+### Fixed
+
+- a bug where layer input and weight quantization functions could not be set using command line arguments
+- a bug where modules could not be imported in OS that use different path separators than '/'
+
 ## [0.1.1] - 2022/01/21
 
 ### Changed
@@ -15,17 +42,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 
 - basic quantized layers
-    - QActivation
-    - QConv
-    - QLinear
+  - QActivation
+  - QConv
+  - QLinear
 - several debug layers
 - resnet, lenet
 - various quantization functions
-    - approxsign
-    - dorefa
-    - sign
-    - steheaviside
-    - swishsign
+  - approxsign
+  - dorefa
+  - sign
+  - steheaviside
+  - swishsign
 - support for cifar10 and mnist
 - general training script for image classification
 - result logger for csv and tensorboard

README.md

@@ -14,55 +14,78 @@ All changes are tracked in the [changelog](CHANGELOG.md).
 ## Installation
 
 Similar to recent versions of [torchvision](https://github.com/pytorch/vision), you should be using Python 3.8 or newer.
+Currently, the only supported installation is pip (a conda package is planned in the future).
 
 ### Pip
 
-If you wish to use a specific version of PyTorch for compatibility with certain devices or CUDA versions,
-we advise on installing the corresponding versions of `pytorch` and `torchvision` first,
+If you wish to use a *specific version* of PyTorch for compatibility with certain devices or CUDA versions,
+we advise on installing the corresponding versions of `pytorch` and `torchvision` first (or afterwards),
 please consult [pytorch's getting started guide](https://pytorch.org/get-started/locally/).
-A good solution to use CUDA 11.x is to install the packages `"torch==1.9.0+cu111" "torchvision==0.10.0+cu111"` first.
 
-Install the package with pip (the `--find-links` option can be removed if torch and torchvision have already been installed):
+Afterwards simply run:
 ```bash
-pip install bitorch --find-links https://download.pytorch.org/whl/torch_stable.html
+pip install bitorch
 ```
 
-To use advanced logging capabilities with [tensorboardX](https://github.com/lanpa/tensorboardX), install the optional dependencies as well:
+Note, that you can also request a specific PyTorch version directly, e.g. for CUDA 11.3:
 ```bash
-pip install "bitorch[opt]" --find-links https://download.pytorch.org/whl/torch_stable.html
+pip install bitorch --extra-index-url https://download.pytorch.org/whl/cu113
+```
+
+To use advanced logging capabilities with [tensorboardX](https://github.com/lanpa/tensorboardX),
+install the optional dependencies as well:
+
+```bash
+pip install "bitorch[opt]"
 ```
 
 #### Local and Development Install Options
 
 The package can also be installed locally for editing and development.
 First, clone the [repository](https://github.com/hpi-xnor/bitorch), then run:
+
 ```bash
-pip install -e . --find-links https://download.pytorch.org/whl/torch_stable.html
+pip install -e .
 ```
 
 To activate advanced logging with Tensorboard and model summary, install the optional dependencies as well:
+
 ```bash
-pip install -e ".[opt]" --find-links https://download.pytorch.org/whl/torch_stable.html
+pip install -e ".[opt]"
 ```
 
-Make sure the _dev_ option is used for (local) development:
-```bash
-pip install -e ".[dev]" --find-links https://download.pytorch.org/whl/torch_stable.html
+### Dali Preprocessing
+
+If you want to use the [Nvidia dali preprocessing library](https://github.com/NVIDIA/DALI),
+e.g. with CUDA 11.x, (currently only supported for imagenet)
+you need to install the `nvidia-dali-cuda110` package by running the following command:
+
+```
+ pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda110
 ```
 
 ### Code formatting and typing
 
+Install the _dev_ requirements for (local) development:
+
+```bash
+pip install -e ".[dev]"
+```
+
 New code should be compatible with Python 3.X versions and be compliant with PEP8. To check the codebase, please run
+
 ```bash
-flake8 --config=setup.cfg .
+flake8
 ```
 
 The codebase has type annotations, please make sure to add type hints if required. We use `mypy` for type checking:
+
 ```bash
 mypy --config-file mypy.ini
 ```
 
 Finally, the tests can be run with:
+
 ```bash
 pytest
 ```

bitorch/__init__.py

@@ -16,8 +16,8 @@
     if file.suffix == ".py" and file.stem != "__init__":
 
         rel_path = Path(os.path.relpath(file, current_dir))
-        import_path = f"{__name__}.{str(rel_path).replace('/', '.')}"
-        import_path = import_path[:import_path.rfind(".")]
+        path_parts = list(rel_path.parent.parts) + [rel_path.stem]
+        import_path = f"{__name__}.{'.'.join(path_parts)}"
         module = import_module(import_path)
 
         for attr_name in dir(module):

bitorch/datasets/__init__.py

@@ -1,3 +1,8 @@
+"""
+This submodule contains data preparation code for some of the datasets used with our models,
+i.e. MNIST, CIFAR 10 and 100 and ImageNet.
+"""
+
 from typing import List, Type
 
 from .base import BasicDataset

bitorch/datasets/base.py

@@ -76,11 +76,9 @@ def get_train_and_test(
         return cls(True, root_directory, download, augmentation), cls(False, root_directory, download)
 
     @classmethod
-    def get_dummy_train_and_test_loaders(cls, batch_size: int) -> Tuple[DummyDataset, DummyDataset]:
-        x_data = torch.zeros((batch_size,) + cls.shape[1:])
-        y_data = torch.zeros((batch_size,), dtype=torch.int64)
-        train_set = DummyDataset((x_data, y_data), batch_size, cls.num_train_samples)
-        val_set = DummyDataset((x_data, y_data), batch_size, cls.num_val_samples)
+    def get_dummy_train_and_test_datasets(cls) -> Tuple[DummyDataset, DummyDataset]:
+        train_set = DummyDataset(cls.shape, cls.num_classes, cls.num_train_samples)  # type: ignore
+        val_set = DummyDataset(cls.shape, cls.num_classes, cls.num_val_samples)  # type: ignore
         return train_set, val_set
 
     def get_dataset_root_directory(self, root_directory_argument: Optional[str]) -> Path:
@@ -98,10 +96,11 @@ def get_dataset_root_directory(self, root_directory_argument: Optional[str]) ->
         if os.environ.get("BITORCH_DATA_HOME") is not None:
             return Path(os.environ.get("BITORCH_DATA_HOME")) / self.name  # type: ignore
 
-        environment_variable_hint = \
-            f" To change this, set '{environment_variable_name}' or 'BITORCH_DATA_HOME' " \
+        environment_variable_hint = (
+            f" To change this, set '{environment_variable_name}' or 'BITORCH_DATA_HOME' "
             f"(in the latter case, the data resides in the folder '{self.name}' in BITORCH_DATA_HOME)."
-
+            f" Some datasets can be downloaded by adding the --download command line argument."
+        )
         if self._download:
             logging.warning("Dataset is being downloaded to the directory './data'." + environment_variable_hint)
             return Path("./data")

bitorch/datasets/dummy_dataset.py

@@ -1,34 +1,23 @@
-from typing import Union, Tuple
-
+from torch.utils.data import Dataset
 import torch
+from typing import Tuple
 
 
-class DummyDataset(object):
+class DummyDataset(Dataset):
     """An iterator that produces repeated dummy data.
     Args:
         data_sample: a data sample that should be produced at each step.
         batch_size: the batch size for storing.
         sample_count: number of `data` samples in the dummy dataset.
     """
 
-    def __init__(self, data_sample: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],
-                 batch_size: int, sample_count: int):
-        self._data_sample = data_sample
+    def __init__(self, data_shape: torch.Size, num_classes: int, sample_count: int) -> None:
+        self._data_sample = torch.zeros(data_shape)
+        self._class_sample = torch.zeros((num_classes,), dtype=torch.int64)
         self._sample_count = sample_count
-        self.batch_size = batch_size
-        self._count = 0
-
-    def __iter__(self) -> "DummyDataset":
-        return DummyDataset(self._data_sample, self.batch_size, self._sample_count)
 
     def __len__(self) -> int:
         return self._sample_count
 
-    def __next__(self) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
-        return self.next()
-
-    def next(self) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
-        if self._count >= self._sample_count:
-            raise StopIteration
-        self._count += 1
-        return self._data_sample
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]:
+        return self._data_sample, self._class_sample

bitorch/datasets/imagenet.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 from torch.utils.data import Dataset
 from torchvision import transforms
@@ -16,9 +17,14 @@ class ImageNet(BasicDataset):
     num_train_samples = 1281167
     num_val_samples = 50000
 
-    def get_dataset(self, download: bool) -> Dataset:
+    def get_data_dir(self) -> Path:
         split = "train" if self.is_train else "val"
         directory = self.root_directory / split
+        return directory
+
+    def get_dataset(self, download: bool) -> Dataset:
+        directory = self.get_data_dir()
+        print("got directory for imagenet:", directory)
         if download and not directory.is_dir():
             raise RuntimeError("ImageNet dataset must be downloaded and prepared manually.")
         return ImageFolder(directory, transform=self.get_transform())

bitorch/layers/__init__.py

@@ -1,3 +1,9 @@
+"""
+This submodule contains adapted pytorch layers that use quantization functions on their weights
+and activations before forwarding them. These layers use the quantization functions specified in the
+quantization submodule.
+"""
+
 from .debug_layers import (
     InputGraphicalDebug,
     InputPrintDebug,
@@ -6,12 +12,15 @@
     ShapePrintDebug
 )
 from .qactivation import QActivation
-from .qconv import QConv1d, QConv2d, QConv3d
-from .qconv_noact import QConv1d_NoAct, QConv2d_NoAct, QConv3d_NoAct
+from .qconv1d import QConv1d, QConv1d_NoAct
+from .qconv2d import QConv2d, QConv2d_NoAct
+from .qconv3d import QConv3d, QConv3d_NoAct
 from .qlinear import QLinear
+from .pact import Pact
+from .qembedding import QEmbedding, QEmbeddingBag
 
 __all__ = [
     "InputGraphicalDebug", "InputPrintDebug", "WeightGraphicalDebug", "WeightPrintDebug",
     "ShapePrintDebug", "QActivation", "QConv1d", "QConv2d", "QConv3d", "QConv1d_NoAct",
-    "QConv2d_NoAct", "QConv3d_NoAct", "QLinear"
+    "QConv2d_NoAct", "QConv3d_NoAct", "QLinear", "QEmbedding", "QEmbeddingBag", "Pact",
 ]