Updated TODO, added comments

Author: aunetx

scripts/activations.py

@@ -2,65 +2,65 @@
 
 
 # Relu
-def relu(y):
+def relu(y) -> np.ndarray:
     return np.maximum(y, 0)
 
 
-def relu_prime(y):
+def relu_prime(y) -> np.ndarray:
     return y > 0
 
 
 # Leaky relu
-def leaky_relu(y):
+def leaky_relu(y) -> np.ndarray:
     return np.where(y > 0, y, y * 0.01)
 
 
-def leaky_relu_prime(y):
+def leaky_relu_prime(y) -> np.ndarray:
     return (y >= 0) + (y < 0)*0.01
 
 
 # Linear
-def linear(y):
+def linear(y) -> np.ndarray:
     return y
 
 
-def linear_prime(y):
+def linear_prime(y) -> np.ndarray:
     return 1
 
 
 # Heavyside
-def heaviside(y):
+def heaviside(y) -> np.ndarray:
     return 1 * (y > 0)
 
 
-def heaviside_prime(y):
+def heaviside_prime(y) -> np.ndarray:
     return 0
 
 
 # Sigmoid
-def sigmoid(y):
+def sigmoid(y) -> np.ndarray:
     return 1 / (1 + np.exp(-y))
 
 
-def sigmoid_prime(y):
+def sigmoid_prime(y) -> np.ndarray:
     return y * (1 - y)
 
 
 # Tanh
-def tanh(y):
+def tanh(y) -> np.ndarray:
     return np.tanh(y)
 
 
-def tanh_prime(y):
+def tanh_prime(y) -> np.ndarray:
     return 1 - y**2
 
 
 # Arctan
-def arctan(y):
+def arctan(y) -> np.ndarray:
     return np.arctan(y)
 
 
-def arctan_prime(y):
+def arctan_prime(y) -> np.ndarray:
     return 1 / y**2 + 1
 
 

scripts/loulou.py

@@ -1,15 +1,15 @@
+from tqdm import tqdm
 import numpy as np
 import json
-import os
-import mnist
 import sys
-from tqdm import tqdm
+import os
 
 import activations
+import mnist
 import utils
 
 
-def feed_forward(X_input, weights, activation_fn):
+def feed_forward(X_input: np.ndarray, weights: list, activation_fn: list) -> np.ndarray:
     """Feed fordward the network
 
     X_input     => input layer
@@ -34,7 +34,7 @@ def feed_forward(X_input, weights, activation_fn):
     return x
 
 
-def grads(x, y_expected, weights, activations_fn, activations_prime):
+def grads(x: np.ndarray, y_expected: np.ndarray, weights: list, activations_fn: list, activations_prime: list) -> np.ndarray:
     """Calculate errors corrections with backward propagation
 
     x           => input layer
@@ -54,7 +54,7 @@ def grads(x, y_expected, weights, activations_fn, activations_prime):
     delta = y[-1] - y_expected
 
     # Calculate error of output weights layer
-    grads = np.empty_like(weights)
+    grads: np.ndarray = np.empty_like(weights)
     grads[-1] = y[-2].T.dot(delta)
 
     # Backward loop
@@ -69,64 +69,91 @@ def grads(x, y_expected, weights, activations_fn, activations_prime):
     return grads / len(x)
 
 
-def train(weights, trX, trY, teX, teY, filename, epochs, batch, learning_rate, save_timeout, reduce_output, activations_fn, activations_prime):
+def train(weights: list, trX: np.ndarray, trY: np.ndarray, teX: np.ndarray, teY: np.ndarray, activations_fn: list, activations_prime: list, filename: np.ndarray, epochs: int, batch: int, learning_rate: float, save_timeout: int, reduce_output: int) -> dict:
     path = os.path.dirname(__file__)
-    accuracy = {}
-    prediction = np.argmax(feed_forward(
+    accuracy = []
+
+    # Make prediction with the untrained network
+    prediction: np.ndarray = np.argmax(feed_forward(
         teX, weights, activations_fn)[-1], axis=1)
-    accuracy[0] = np.mean(prediction == np.argmax(teY, axis=1))
-    if reduce_output < 2:
+    accuracy.append(np.mean(prediction == np.argmax(teY, axis=1)))
+
+    if reduce_output <= 1:
         print('Accuracy of epoch 0 :', accuracy[0])
-    if reduce_output == 2:
+    elif reduce_output == 2:
         print(0, accuracy[0])
+
     if epochs < 0:
         epochs = 99999999999
+
+    # Epochs loop
     for i in range(epochs):
         if reduce_output < 1:
+
             pbar = tqdm(range(0, len(trX), batch))
         else:
             pbar = range(0, len(trX), batch)
+
+        # Batches loop
         for j in pbar:
             if reduce_output < 1:
                 pbar.set_description("Processing epoch %s" % (i+1))
 
+            # Select training data
             X, Y = trX[j:j+batch], trY[j:j+batch]
+
+            # Correct the network
             weights -= learning_rate * \
                 grads(X, Y, weights, activations_fn, activations_prime)
+
+        # Make prediction for epoch
         prediction = np.argmax(feed_forward(
             teX, weights, activations_fn)[-1], axis=1)
-        accuracy[i+1] = np.mean(prediction == np.argmax(teY, axis=1))
+        accuracy.append(np.mean(prediction == np.argmax(teY, axis=1)))
+
         if reduce_output < 2:
             print('Accuracy of epoch', i+1, ':', accuracy[i+1])
         if reduce_output == 2:
             print(i+1, accuracy[i+1])
+
+        # Save temp file if set so
         if filename:
             if save_timeout > 0:
                 if i % save_timeout == 0:
                     temp_filename = '../trains/temp/' + \
                         filename + '_epoch_' + str(i) + '.npy'
                     temp_filename = os.path.join(path, temp_filename)
                     utils.save(weights, temp_filename, reduce_output)
+
+    # Save final file
     if filename:
         filename = os.path.join(path, '../trains/' + filename + '.npy')
         utils.save(weights, filename, reduce_output)
+
     return accuracy
 
 
-def runTrain(params, architecture, file=None):
-    params = json.loads(params)
-    epochs = params['epochs']
-    batch = params['batch']
-    learning_rate = params['learning_rate']
-    save_timeout = params['save_timeout']
-    reduce_output = params['reduce_output']
+def runTrain(params: dict, architecture: list, file=None) -> dict:
+    params: dict = json.loads(params)
+    epochs: int = params['epochs']
+    batch: int = params['batch']
+    learning_rate: float = params['learning_rate']
+    save_timeout: int = params['save_timeout']
+    reduce_output: int = params['reduce_output']
     activations_arch, primes_arch = activations.listToActivations(
         params['activations'], architecture)
 
+    # Print network visualization
     if reduce_output < 1:
         utils.print_network_visualization(
             architecture, activations_arch, epochs, batch, learning_rate)
 
+    # Load data
+    #   TODO do not load arbitrary data
     trX, trY, teX, teY = mnist.load_data()
+
+    # Init weights
     weights = [np.random.randn(*w) * 0.1 for w in architecture]
-    return train(weights, trX, trY, teX, teY, file, epochs, batch, learning_rate, save_timeout, reduce_output, activations_arch, primes_arch)
+
+    # Train network
+    return train(weights, trX, trY, teX, teY, activations_arch, primes_arch, file, epochs, batch, learning_rate, save_timeout, reduce_output)

scripts/mnist.py

@@ -9,7 +9,9 @@
 # Download and import the MNIST dataset from Yann LeCun's website.
 # Reserve 10,000 examples from the training set for validation.
 # Each image is an array of 784 (28x28) float values  from 0 (white) to 1 (black).
-def load_data(one_hot=True, reshape=None, validation_size=10000):
+
+
+def load_data(one_hot: bool = True, reshape: bool = None, validation_size: int = 10000) -> (np.ndarray, np.ndarray, np.ndarray, np.ndarray):
     x_tr = load_images('train-images-idx3-ubyte.gz')
     y_tr = load_labels('train-labels-idx1-ubyte.gz')
     x_te = load_images('t10k-images-idx3-ubyte.gz')
@@ -26,26 +28,30 @@ def load_data(one_hot=True, reshape=None, validation_size=10000):
 
     return x_tr, y_tr, x_te, y_te
 
-def load_images(filename):
+
+def load_images(filename: str) -> np.ndarray:
     maybe_download(filename)
     with gzip.open(path+filename, 'rb') as f:
         data = np.frombuffer(f.read(), np.uint8, offset=16)
     return data.reshape(-1, 28 * 28) / np.float32(256)
 
-def load_labels(filename):
+
+def load_labels(filename: str) -> np.ndarray:
     maybe_download(filename)
     with gzip.open(path+filename, 'rb') as f:
         data = np.frombuffer(f.read(), np.uint8, offset=8)
     return data
 
+
 # Download the file, unless it's already here.
-def maybe_download(filename):
+def maybe_download(filename: str) -> None:
     if not os.path.exists(path+filename):
         print('Please wait while downloading training dataset.')
         from urllib.request import urlretrieve
         print("Downloading %s" % filename)
         urlretrieve(DATA_URL + filename, path+filename)
 
+
 # Convert class labels from scalars to one-hot vectors.
-def to_one_hot(labels, num_classes=10):
+def to_one_hot(labels: np.ndarray, num_classes: int = 10) -> np.ndarray:
     return np.eye(num_classes)[labels]

scripts/run.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import numpy as np
 import sys
 import matplotlib.image as image

scripts/train.py

@@ -1,8 +1,11 @@
-from loulou import runTrain
-from utils import listToArch
+#!/usr/bin/env python3
+
 import json
 import argparse
 
+from loulou import runTrain
+from utils import listToArch
+
 if __name__ == '__main__':
 
     params = {}

scripts/utils.py

@@ -2,20 +2,20 @@
 import json
 
 
-def save(weights, filename, reduce_output):
+def save(weights: list, filename: str, reduce_output: int) -> None:
     np.save(filename, weights)
     if reduce_output < 2:
         print('Data saved successfully into ', filename)
 
 
-def convertJson(pred):
+def convertJson(pred: np.ndarray) -> str:
     out = {}
     out['hot_prediction'] = list(pred)
     out['prediction'] = int(np.argmax(pred))
     return json.dumps(out)
 
 
-def listToArch(list):
+def listToArch(list: list) -> list:
     arch = []
     id = 0
     for hl in list:
@@ -33,7 +33,7 @@ def listToArch(list):
     return arch
 
 
-def print_network_visualization(architecture, activations_arch, epochs, batch, learning_rate):
+def print_network_visualization(architecture: list, activations_arch: list, epochs: int, batch: int, learning_rate: float) -> None:
     print('Network has', len(architecture) - 1, 'hidden layers :')
 
     print('     layer [0]  -->  784   neurons, inputs')

todo.md

@@ -1,4 +1,8 @@
 # To-do
 
-- Handle args in run.py just like in train.py
-- Handle others types of data
+- Update README
+- Handle args in `run.py` just like in `train.py`
+- Write descriptions for functions
+- Handle others types of data
+- Save trainings to an archive with datas : `weights matrix`, `activations arch`, [`hyperparameters`], [`accuracy`]
+- Rewrite as object ? (OOP)