Support for scikit neural networks and renaming omlt.onnx.py

docs/notebooks/neuralnet/mnist_example_convolutional.ipynb

@@ -53,7 +53,7 @@
     "#omlt for interfacing our neural network with pyomo\n",
     "from omlt import OmltBlock\n",
     "from omlt.neuralnet import FullSpaceNNFormulation\n",
-    "from omlt.io.onnx import write_onnx_model_with_bounds, load_onnx_neural_network_with_bounds"
+    "from omlt.io.onnx_reader import write_onnx_model_with_bounds, load_onnx_neural_network_with_bounds"
    ]
   },
   {
@@ -659,4 +659,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

docs/notebooks/neuralnet/mnist_example_dense.ipynb

@@ -52,7 +52,7 @@
     "#omlt for interfacing our neural network with pyomo\n",
     "from omlt import OmltBlock\n",
     "from omlt.neuralnet import FullSpaceNNFormulation\n",
-    "from omlt.io.onnx import write_onnx_model_with_bounds, load_onnx_neural_network_with_bounds"
+    "from omlt.io.onnx_reader import write_onnx_model_with_bounds, load_onnx_neural_network_with_bounds"
    ]
   },
   {
@@ -742,4 +742,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

src/omlt/io/__init__.py

@@ -1,2 +1,2 @@
-from omlt.io.onnx import load_onnx_neural_network, write_onnx_model_with_bounds, load_onnx_neural_network_with_bounds
+from omlt.io.onnx_reader import load_onnx_neural_network, write_onnx_model_with_bounds, load_onnx_neural_network_with_bounds
 from omlt.io.keras_reader import load_keras_sequential

src/omlt/io/onnx_parser.py

@@ -10,7 +10,7 @@
 )
 
 
-_ACTIVATION_OP_TYPES = ["Relu", "Sigmoid", "LogSoftmax"]
+_ACTIVATION_OP_TYPES = ["Relu", "Sigmoid", "LogSoftmax", "Tanh"]
 
 
 class NetworkParser:
@@ -181,6 +181,10 @@ def _consume_dense_nodes(self, node, next_nodes):
             node_biases = self._initializers[in_1]
 
         assert len(node_weights.shape) == 2
+
+        # Flatten biases array as some APIs (scikit) store biases as (1, n) instead of (n,)
+        node_biases = node_biases.flatten()
+
         assert node_weights.shape[1] == node_biases.shape[0]
         assert len(node.output) == 1
 
@@ -339,7 +343,12 @@ def _consume_reshape_nodes(self, node, next_nodes):
         assert len(node.input) == 2
         [in_0, in_1] = list(node.input)
         input_layer = self._node_map[in_0]
-        new_shape = self._constants[in_1]
+
+        if in_1 in self._constants:
+            new_shape = self._constants[in_1]
+        else:
+            new_shape = self._initializers[in_1]
+
         output_size = np.empty(input_layer.output_size).reshape(new_shape).shape
         transformer = IndexMapper(input_layer.output_size, list(output_size))
         self._node_map[node.output[0]] = (transformer, input_layer)

src/omlt/io/sklearn_reader.py

@@ -0,0 +1,67 @@
+from skl2onnx.common.data_types import FloatTensorType
+from skl2onnx import convert_sklearn
+from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler, StandardScaler
+from sklearn.preprocessing import RobustScaler
+from omlt.scaling import OffsetScaling
+from omlt.io.onnx_reader import load_onnx_neural_network
+import onnx
+
+def parse_sklearn_scaler(sklearn_scaler):
+
+    if isinstance(sklearn_scaler, StandardScaler):
+        offset = sklearn_scaler.mean_
+        factor = sklearn_scaler.scale_
+
+    elif isinstance(sklearn_scaler, MaxAbsScaler):
+        factor = sklearn_scaler.scale_
+        offset = factor*0
+
+    elif isinstance(sklearn_scaler, MinMaxScaler):
+        factor = sklearn_scaler.data_max_ - sklearn_scaler.data_min_
+        offset = sklearn_scaler.data_min_
+
+    elif isinstance(sklearn_scaler, RobustScaler):
+        factor = sklearn_scaler.scale_
+        offset = sklearn_scaler.center_
+
+    else:
+        raise(ValueError("Scaling object provided is not currently supported. Only linear scalers are supported."
+                         "Supported scalers include StandardScaler, MinMaxScaler, MaxAbsScaler, and RobustScaler"))
+
+    return offset, factor
+
+def convert_sklearn_scalers(sklearn_input_scaler, sklearn_output_scaler):
+
+    #Todo: support only scaling input or output?
+
+    offset_inputs, factor_inputs = parse_sklearn_scaler(sklearn_input_scaler)
+    offset_outputs, factor_ouputs = parse_sklearn_scaler(sklearn_output_scaler)
+
+    return OffsetScaling(offset_inputs=offset_inputs, factor_inputs=factor_inputs,
+                         offset_outputs=offset_outputs, factor_outputs=factor_ouputs)
+
+def load_sklearn_MLP(model, scaling_object=None, input_bounds=None, initial_types=None):
+
+    # Assume float inputs if no types are supplied to the model
+    if initial_types is None:
+        initial_types = [('float_input', FloatTensorType([None, model.n_features_in_]))]
+
+    onx = convert_sklearn(model, initial_types=initial_types, target_opset=12)
+
+    # Remove initial cast layer created by sklearn2onnx
+    graph = onx.graph
+    node1 = graph.node[0]
+    graph.node.remove(node1)
+    new_node = onnx.helper.make_node(
+        'MatMul',
+        name="MatMul",
+        inputs=['float_input', 'coefficient'],
+        outputs=['mul_result']
+    )
+    graph.node.insert(0, new_node)
+
+    # Replace old MatMul node with new node with correct input name
+    node2 = graph.node[1]
+    graph.node.remove(node2)
+
+    return load_onnx_neural_network(onx, scaling_object, input_bounds)

tests/io/test_onnx_parser.py

@@ -2,7 +2,7 @@
 import onnx
 import numpy as np
 
-from omlt.io.onnx import load_onnx_neural_network
+from omlt.io.onnx_reader import load_onnx_neural_network
 
 
 def test_linear_131(datadir):

tests/io/test_sklearn.py

@@ -0,0 +1,164 @@
+from omlt.scaling import OffsetScaling
+from omlt.block import OmltBlock
+from omlt.io.sklearn_reader import convert_sklearn_scalers
+from omlt.io.sklearn_reader import load_sklearn_MLP
+from omlt.neuralnet import FullSpaceNNFormulation
+from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler, StandardScaler, RobustScaler
+from scipy.stats import iqr
+from pyomo.environ import *
+import numpy as np
+import json
+import pickle
+
+def test_sklearn_scaler_conversion():
+    X = np.array(
+        [[42, 10, 29],
+         [12, 19, 15]]
+    )
+
+    Y = np.array(
+        [[1, 2],
+         [3, 4]]
+    )
+
+    # Create sklearn scalers
+    xMinMax = MinMaxScaler()
+    xMaxAbs = MaxAbsScaler()
+    xStandard = StandardScaler()
+    xRobust = RobustScaler()
+
+    yMinMax = MinMaxScaler()
+    yMaxAbs = MaxAbsScaler()
+    yStandard = StandardScaler()
+    yRobust = RobustScaler()
+
+    sklearn_scalers = [(xMinMax, yMinMax), (xMaxAbs, yMaxAbs), (xStandard, yStandard), (xRobust, yRobust)]
+    for scalers in sklearn_scalers:
+        scalers[0].fit(X)
+        scalers[1].fit(Y)
+
+    # Create OMLT scalers using OMLT function
+    MinMaxOMLT = convert_sklearn_scalers(xMinMax, yMinMax)
+    MaxAbsOMLT = convert_sklearn_scalers(xMaxAbs, yMaxAbs)
+    StandardOMLT = convert_sklearn_scalers(xStandard, yStandard)
+    RobustOMLT = convert_sklearn_scalers(xRobust, yRobust)
+
+    omlt_scalers = [MinMaxOMLT, MaxAbsOMLT, StandardOMLT, RobustOMLT]
+
+    # Generate test data
+    x = {0: 10, 1: 29, 2: 42}
+    y = {0: 2, 1: 1}
+
+    # Test Scalers
+    for i in range(len(omlt_scalers)):
+        x_s_omlt = omlt_scalers[i].get_scaled_input_expressions(x)
+        y_s_omlt = omlt_scalers[i].get_scaled_output_expressions(y)
+
+        x_s_sklearn = sklearn_scalers[i][0].transform([list(x.values())])[0]
+        y_s_sklearn = sklearn_scalers[i][1].transform([list(y.values())])[0]
+
+        np.testing.assert_almost_equal(list(x_s_omlt.values()), list(x_s_sklearn))
+        np.testing.assert_almost_equal(list(y_s_omlt.values()), list(y_s_sklearn))
+
+def test_sklearn_offset_equivalence():
+    X = np.array(
+        [[42, 10, 29],
+         [12, 19, 15]]
+    )
+
+    Y = np.array(
+        [[1, 2],
+         [3, 4]]
+    )
+
+    # Get scaling factors for OffsetScaler
+    xmean = X.mean(axis=0)
+    xstd = X.std(axis=0)
+    xmax = X.max(axis=0)
+    absxmax = abs(X).max(axis=0)
+    xmin = X.min(axis=0)
+    xminmax = xmax-xmin
+    xmedian = np.median(X, axis=0)
+    xiqr = iqr(X, axis=0)
+
+    ymean = Y.mean(axis=0)
+    ystd = Y.std(axis=0)
+    ymax = Y.max(axis=0)
+    absymax = abs(Y).max(axis=0)
+    ymin = Y.min(axis=0)
+    yminmax = ymax-ymin
+    ymedian = np.median(Y, axis=0)
+    yiqr = iqr(Y, axis=0)
+
+    # Create sklearn scalers
+    xMinMax = MinMaxScaler()
+    xMaxAbs = MaxAbsScaler()
+    xStandard = StandardScaler()
+    xRobust = RobustScaler()
+
+    yMinMax = MinMaxScaler()
+    yMaxAbs = MaxAbsScaler()
+    yStandard = StandardScaler()
+    yRobust = RobustScaler()
+
+    sklearn_scalers = [(xMinMax, yMinMax), (xMaxAbs, yMaxAbs), (xStandard, yStandard), (xRobust, yRobust)]
+    for scalers in sklearn_scalers:
+        scalers[0].fit(X)
+        scalers[1].fit(Y)
+
+    # Create OMLT scalers manually
+    MinMaxOMLT = OffsetScaling(offset_inputs=xmin, factor_inputs=xminmax, offset_outputs=ymin, factor_outputs=yminmax)
+    MaxAbsOMLT = OffsetScaling(offset_inputs=[0]*3, factor_inputs=absxmax, offset_outputs=[0]*2, factor_outputs=absymax)
+    StandardOMLT = OffsetScaling(offset_inputs=xmean, factor_inputs=xstd, offset_outputs=ymean, factor_outputs=ystd)
+    RobustOMLT = OffsetScaling(offset_inputs=xmedian, factor_inputs=xiqr, offset_outputs=ymedian, factor_outputs=yiqr)
+
+    omlt_scalers = [MinMaxOMLT, MaxAbsOMLT, StandardOMLT, RobustOMLT]
+
+    # Generate test data
+    x = {0: 10, 1: 29, 2: 42}
+    y = {0: 2, 1: 1}
+
+    # Test Scalers
+    for i in range(len(omlt_scalers)):
+        x_s_omlt = omlt_scalers[i].get_scaled_input_expressions(x)
+        y_s_omlt = omlt_scalers[i].get_scaled_output_expressions(y)
+
+        x_s_sklearn = sklearn_scalers[i][0].transform([list(x.values())])[0]
+        y_s_sklearn = sklearn_scalers[i][1].transform([list(y.values())])[0]
+
+        np.testing.assert_almost_equal(list(x_s_omlt.values()), list(x_s_sklearn))
+        np.testing.assert_almost_equal(list(y_s_omlt.values()), list(y_s_sklearn))
+
+def test_sklearn_model(datadir):
+    nn_names = ["sklearn_identity_131", "sklearn_logistic_131", "sklearn_tanh_131"]
+
+    # Test each nn
+    for nn_name in nn_names:
+        nn = pickle.load(open(datadir.file(nn_name+".pkl"), 'rb'))
+
+        with open(datadir.file(nn_name+"_bounds"), 'r') as f:
+            bounds = json.load(f)
+
+            # Convert to omlt format
+            xbounds = {int(i): tuple(bounds[i]) for i in bounds}
+
+        net = load_sklearn_MLP(nn, input_bounds=xbounds)
+        formulation = FullSpaceNNFormulation(net)
+
+        model = ConcreteModel()
+        model.nn = OmltBlock()
+        model.nn.build_formulation(formulation)
+
+        @model.Objective()
+        def obj(mdl):
+            return 1
+
+        x = [(xbounds[i][0]+xbounds[i][1])/2.0 for i in range(2)]
+        for i in range(len(x)):
+            model.nn.inputs[i].fix(x[i])
+
+        result = SolverFactory("ipopt").solve(model, tee=False)
+        yomlt = [value(model.nn.outputs[0]), value(model.nn.outputs[1])]
+
+        ysklearn = nn.predict([x])[0]
+        np.testing.assert_almost_equal(list(yomlt), list(ysklearn))

tests/models/sklearn_identity_131_bounds

@@ -0,0 +1 @@
+{"0": [-3.2412673400690726, 2.3146585666735087], "1": [-1.9875689146008928, 3.852731490654721]}

tests/models/sklearn_logistic_131_bounds

@@ -0,0 +1 @@
+{"0": [-3.2412673400690726, 2.3146585666735087], "1": [-1.9875689146008928, 3.852731490654721]}

tests/models/sklearn_tanh_131_bounds

@@ -0,0 +1 @@
+{"0": [-3.2412673400690726, 2.3146585666735087], "1": [-1.9875689146008928, 3.852731490654721]}

tests/neuralnet/test_onnx.py

@@ -1,5 +1,5 @@
 import tempfile
-from omlt.io.onnx import load_onnx_neural_network, load_onnx_neural_network_with_bounds, write_onnx_model_with_bounds
+from omlt.io.onnx_reader import load_onnx_neural_network, load_onnx_neural_network_with_bounds, write_onnx_model_with_bounds
 import onnx
 import onnxruntime as ort
 import numpy as np

tests/neuralnet/test_relu.py

@@ -2,7 +2,7 @@
 import numpy as np
 
 from omlt.block import OmltBlock
-from omlt.io.onnx import load_onnx_neural_network_with_bounds
+from omlt.io.onnx_reader import load_onnx_neural_network_with_bounds
 from omlt.neuralnet import FullSpaceNNFormulation, ReluBigMFormulation, ReluComplementarityFormulation, ReluPartitionFormulation
 from omlt.neuralnet.activations import ComplementarityReLUActivation