Merging some old code before switching from master to main

examples/plot_compare_optimizers.py

@@ -22,7 +22,7 @@
 from celer import LogisticRegression
 from libsvmdata.datasets import fetch_libsvm
 
-from sparse_ho import ImplicitForward
+from sparse_ho import Implicit
 from sparse_ho.ho import grad_search
 from sparse_ho.utils import Monitor
 from sparse_ho.models import SparseLogreg
@@ -97,7 +97,7 @@
     criterion = HeldOutLogistic(idx_train, idx_val)
 
     monitor_grad = Monitor()
-    algo = ImplicitForward(tol_jac=tol, n_iter_jac=1000)
+    algo = Implicit()
 
     optimizer = optimizers[optimizer_name]
     grad_search(

examples/plot_held_out_enet.py

@@ -21,7 +21,7 @@
 from celer.datasets import make_correlated_data
 from sklearn.metrics import mean_squared_error
 
-from sparse_ho import ImplicitForward
+from sparse_ho import Implicit
 from sparse_ho.criterion import HeldOutMSE
 from sparse_ho.models import ElasticNet
 from sparse_ho.ho import grad_search
@@ -98,7 +98,7 @@
 alpha0 = np.array([alpha_max * 0.9, alpha_max * 0.9])
 model = ElasticNet(estimator=estimator)
 criterion = HeldOutMSE(idx_train, idx_val)
-algo = ImplicitForward(tol_jac=1e-3, n_iter_jac=100, max_iter=max_iter)
+algo = Implicit(max_iter=max_iter)
 optimizer = GradientDescent(
     n_outer=n_outer, tol=tol, p_grad_norm=1.5, verbose=True)
 grad_search(

examples/plot_held_out_lasso.py

@@ -24,7 +24,7 @@
 
 from sparse_ho.models import Lasso
 from sparse_ho.criterion import HeldOutMSE
-from sparse_ho import ImplicitForward
+from sparse_ho import Implicit
 from sparse_ho.utils import Monitor
 from sparse_ho.utils_plot import discrete_cmap
 from sparse_ho.ho import grad_search
@@ -87,7 +87,7 @@
 t0 = time.time()
 model = Lasso(estimator=estimator)
 criterion = HeldOutMSE(idx_train, idx_val)
-algo = ImplicitForward()
+algo = Implicit()
 monitor_grad = Monitor()
 optimizer = LineSearch(n_outer=10, tol=tol)
 grad_search(

examples/plot_lassoCV.py

@@ -24,7 +24,7 @@
 from sklearn.linear_model import LassoCV
 from sklearn.model_selection import KFold
 
-from sparse_ho import ImplicitForward, grad_search
+from sparse_ho import Implicit, grad_search
 from sparse_ho.models import Lasso
 from sparse_ho.criterion import HeldOutMSE, CrossVal
 from sparse_ho.optimizers import GradientDescent
@@ -83,7 +83,7 @@
 alpha0 = 0.9 * alpha_max
 monitor_grad = Monitor()
 cross_val_criterion = CrossVal(criterion, cv=kf)
-algo = ImplicitForward()
+algo = Implicit()
 optimizer = GradientDescent(n_outer=10, tol=tol)
 grad_search(
     algo, cross_val_criterion, model, optimizer, X, y, alpha0,

examples/plot_sparse_log_reg.py

@@ -27,7 +27,7 @@
 from sparse_ho.utils import Monitor
 from sparse_ho.models import SparseLogreg
 from sparse_ho.criterion import HeldOutLogistic
-from sparse_ho import ImplicitForward
+from sparse_ho import Implicit
 from sparse_ho.grid_search import grid_search
 from sparse_ho.optimizers import GradientDescent
 from sparse_ho.utils_plot import discrete_cmap
@@ -101,7 +101,7 @@
 criterion = HeldOutLogistic(idx_train, idx_val)
 
 monitor_grad = Monitor()
-algo = ImplicitForward(tol_jac=tol, n_iter_jac=1000)
+algo = Implicit()
 
 optimizer = GradientDescent(n_outer=10, tol=tol)
 grad_search(

examples/plot_use_callback.py

@@ -21,7 +21,7 @@
 
 from sparse_ho.models import Lasso
 from sparse_ho.criterion import HeldOutMSE
-from sparse_ho import ImplicitForward
+from sparse_ho import Implicit
 from sparse_ho.utils import Monitor
 from sparse_ho.ho import grad_search
 from sparse_ho.optimizers import LineSearch
@@ -70,7 +70,7 @@ def callback(val, grad, mask, dense, alpha):
 # ---------------------------------------
 model = Lasso(estimator=estimator)
 criterion = HeldOutMSE(idx_train, idx_val)
-algo = ImplicitForward()
+algo = Implicit()
 # use Monitor(callback) with your custom callback
 monitor = Monitor(callback=callback)
 optimizer = LineSearch(n_outer=30)

examples/plot_wlasso.py

@@ -26,7 +26,7 @@
 
 from sparse_ho.models import WeightedLasso
 from sparse_ho.criterion import HeldOutMSE, CrossVal
-from sparse_ho import ImplicitForward
+from sparse_ho import Implicit
 from sparse_ho.utils import Monitor
 from sparse_ho.ho import grad_search
 from sparse_ho.optimizers import GradientDescent
@@ -79,7 +79,7 @@
 model = WeightedLasso(estimator=estimator)
 sub_criterion = HeldOutMSE(idx_train, idx_val)
 criterion = CrossVal(sub_criterion, cv=cv)
-algo = ImplicitForward()
+algo = Implicit()
 monitor = Monitor()
 optimizer = GradientDescent(
     n_outer=100, tol=1e-7, verbose=True, p_grad_norm=1.9)

expes/expe_cvxpy/figure_cvxpy.py

@@ -14,8 +14,8 @@
 
 current_palette = sns.color_palette("colorblind")
 dict_method = {}
-dict_method["forward"] = 'PCD Forward Iterdiff'
-dict_method["backward"] = 'PCD Backward Iterdiff'
+dict_method["forward"] = 'Forward-mode PCD'
+dict_method["backward"] = 'Reverse-mode PCD'
 dict_method['cvxpy'] = 'Cvxpylayers'
 
 dict_div_alphas = {}

expes/expe_enet/figure_enet_pred.py

@@ -6,8 +6,8 @@
     discrete_color, dict_color, dict_color_2Dplot, dict_markers,
     dict_method, dict_title, configure_plt)
 
-# save_fig = False
-save_fig = True
+save_fig = False
+# save_fig = True
 # fig_dir = "./"
 # fig_dir_svg = "./"
 fig_dir = "../../../CD_SUGAR/tex/journal/prebuiltimages/"
@@ -28,14 +28,17 @@
 dict_marker_size["forward"] = 4
 dict_marker_size["implicit_forward"] = 10
 dict_marker_size["implicit_forward_approx"] = 10
+dict_marker_size["implicit"] = 10
+dict_marker_size["implicit_approx"] = 10
 dict_marker_size["fast_iterdiff"] = 4
-dict_marker_size['implicit'] = 4
 dict_marker_size['grid_search'] = 5
 dict_marker_size['bayesian'] = 10
 dict_marker_size['random'] = 5
 dict_marker_size['lhs'] = 4
 
 dict_s = {}
+dict_s["implicit"] = 50
+dict_s["implicit_approx"] = 70
 dict_s["implicit_forward"] = 50
 dict_s["implicit_forward_approx"] = 70
 dict_s['grid_search'] = 40
@@ -69,6 +72,7 @@
 
 markersize = 8
 
+# dataset_names = ["rcv1_train", "real-sim", "real-sim"]
 dataset_names = ["rcv1_train", "real-sim", "news20"]
 
 
@@ -83,33 +87,36 @@
     3, len(dataset_names), sharex=False, sharey=False, figsize=[11, 10],
     )
 
+dict_idx = {}
+dict_idx['grid_search'] = 0
+dict_idx['bayesian'] = 1
+dict_idx['implicit_forward_approx'] = 2
+dict_idx['implicit_approx'] = 2
 
 model_name = "enet"
+methods = [
+    # "implicit", "implicit_approx", 'grid_search',
+    "implicit", 'grid_search',
+    'random', 'bayesian', "implicit_approx"]
 
 for idx, dataset in enumerate(dataset_names):
-    df_data = pd.read_pickle("results/%s_%s.pkl" % (model_name, dataset))
-    df_data = df_data[df_data['tolerance_decrease'] == 'constant']
+    for method in methods:
+        df_data = pd.read_pickle(
+            "results/%s_%s_%s.pkl" % (model_name, dataset, method))
 
-    methods = df_data['method']
-    times = df_data['times']
-    objs = df_data['objs']
-    all_alphas = df_data['alphas']
-    alpha_max = df_data['alpha_max'].to_numpy()[0]
-    tols = df_data['tolerance_decrease']
+        time = df_data['times'].to_numpy()[0]
+        obj = np.array(df_data['objs'].to_numpy()[0])
+        alphas = np.array(df_data['alphas'].to_numpy()[0])
+        alpha_max = df_data['alpha_max'].to_numpy()[0]
 
-    min_objs = np.infty
-    for obj in objs:
-        min_objs = min(min_objs, obj.min())
+        lines = []
 
-    lines = []
+        axarr_test.flat[idx].set_xlim(0, dict_xmax[model_name, dataset])
+        axarr_test.flat[idx].set_xlabel("Time (s)", fontsize=fontsize)
 
-    axarr_test.flat[idx].set_xlim(0, dict_xmax[model_name, dataset])
-    axarr_test.flat[idx].set_xlabel("Time (s)", fontsize=fontsize)
-
-    E0 = df_data.objs.to_numpy()[2][0]
-    for _, (time, obj, alphas, method, _) in enumerate(
-            zip(times, objs, all_alphas, methods, tols)):
+        E0 = 1
         if method == 'grid_search':
+            min_objs = obj.min()
             alpha1D = np.unique(alphas)
             alpha1D.sort()
             alpha1D = np.log(np.flip(alpha1D) / alpha_max)
@@ -124,54 +131,35 @@
                     X, Y, (results.T - min_objs) / min_objs, levels=levels,
                     cmap=cmap, linewidths=0.5)
 
-    for _, (time, obj, alphas, method, _) in enumerate(
-            zip(times, objs, all_alphas, methods, tols)):
         marker = dict_markers[method]
         n_outer = len(obj)
         s = dict_s[method]
         color = discrete_color(n_outer, dict_color_2Dplot[method])
-        if method == 'grid_search':
-            i = 0
-            axarr_grad[i, idx].scatter(
-                np.log(alphas[:, 0] / alpha_max),
-                np.log(alphas[:, 1] / alpha_max),
-                s=s, color=color,
-                marker=dict_markers[method], label="todo", clip_on=False)
-        elif method == 'bayesian':
-            i = 1
-            axarr_grad[i, idx].scatter(
-                np.log(alphas[:, 0] / alpha_max),
-                np.log(alphas[:, 1] / alpha_max),
-                s=s, color=color,
-                marker=dict_markers[method], label="todo", clip_on=False)
-        elif method == 'implicit_forward_approx':
-            i = 2
-            axarr_grad[i, idx].scatter(
+
+        if method in dict_idx:
+            axarr_grad[dict_idx[method], idx].scatter(
                 np.log(alphas[:, 0] / alpha_max),
                 np.log(alphas[:, 1] / alpha_max),
                 s=s, color=color,
                 marker=dict_markers[method], label="todo", clip_on=False)
-        else:
-            pass
-        axarr_grad[i, 0].set_ylabel(
-            "%s \n" % dict_method[method] + r"$\lambda_2 - \lambda_{\max}$",
-            fontsize=fontsize)
-
-    for i in range(3):
-        axarr_grad[i, idx].set_xlim((alpha1D.min(), alpha1D.max()))
-        axarr_grad[i, idx].set_ylim((alpha1D.min(), alpha1D.max()))
+            axarr_grad[dict_idx[method], 0].set_ylabel(
+                "%s \n" % dict_method[method] + r"$\lambda_2 - \lambda_{\max}$",
+                fontsize=fontsize)
 
-    for _, (time, obj, method, _) in enumerate(
-            zip(times, objs, methods, tols)):
         marker = dict_markers[method]
         markersize = dict_marker_size[method]
-        obj = [np.min(obj[:k]) for k in np.arange(len(obj)) + 1]
+        obj = np.array([np.min(obj[:k]) for k in np.arange(len(obj)) + 1])
         lines.append(
             axarr_val.flat[idx].plot(
                 time, obj / E0,
                 color=dict_color[method], label="%s" % (dict_method[method]),
                 marker=marker, markersize=markersize,
                 markevery=dict_markevery[dataset]))
+
+    for i in range(3):
+        axarr_grad[i, idx].set_xlim((alpha1D.min(), alpha1D.max()))
+        axarr_grad[i, idx].set_ylim((alpha1D.min(), alpha1D.max()))
+
     axarr_val.flat[idx].set_xlim(0, dict_xmax[model_name, dataset])
     axarr_val.flat[idx].set_xlabel("Time (s)", fontsize=fontsize)