BENCHMARKS updated saga benchmark to account for new dtype implementation

NelleV · NelleV · commit 6aa7b93b8cc5 · 2018-05-31T11:53:15.000-07:00
diff --git a/benchmarks/bench_saga.py b/benchmarks/bench_saga.py
@@ -5,14 +5,13 @@
 """
 import json
 import time
-from os.path import expanduser
 
 import matplotlib.pyplot as plt
 import numpy as np
 
 from sklearn.datasets import fetch_rcv1, load_iris, load_digits, \
     fetch_20newsgroups_vectorized
-from sklearn.externals.joblib import delayed, Parallel, Memory
+from sklearn.externals.joblib import delayed, Parallel
 from sklearn.linear_model import LogisticRegression
 from sklearn.metrics import log_loss
 from sklearn.model_selection import train_test_split
@@ -70,11 +69,15 @@ def fit_single(solver, X, y, penalty='l2', single_target=True, C=1,
                                     multi_class=multi_class,
                                     C=C,
                                     penalty=penalty,
-                                    fit_intercept=False, tol=1e-24,
+                                    fit_intercept=False, tol=0,
                                     max_iter=this_max_iter,
                                     random_state=42,
                                     )
+
+        # Makes cpu cache even for all fit calls
+        X_train.max()
         t0 = time.clock()
+
         lr.fit(X_train, y_train)
         train_time = time.clock() - t0
 
@@ -98,6 +101,7 @@ def fit_single(solver, X, y, penalty='l2', single_target=True, C=1,
         accuracies.append(accuracy)
         times.append(train_time)
     return lr, times, train_scores, test_scores, accuracies
+    return True
 
 
 def _predict_proba(lr, X):
@@ -107,14 +111,13 @@ def _predict_proba(lr, X):
     return softmax(pred)
 
 
-def exp(solvers, penalties, single_target, dtypes=[np.float64, np.float32],
+def exp(solvers, penalties, single_target,
         n_samples=30000, max_iter=20,
         dataset='rcv1', n_jobs=1, skip_slow=False):
-    mem = Memory(cachedir=expanduser('~/cache'), verbose=0)
-    dtypes_mapping = {"float32": np.float32,
-                      "float64": np.float64}
-    if np.float32 in dtypes and "lightning" in solvers:
-        raise ValueError("Lightning does not support np.float32")
+    dtypes_mapping = {
+                      "float64": np.float64,
+                      "float32": np.float32,
+                      }
 
     if dataset == 'rcv1':
         rcv1 = fetch_rcv1()
@@ -154,12 +157,12 @@ def exp(solvers, penalties, single_target, dtypes=[np.float64, np.float32],
             y_n[y <= 16] = 0
             y = y_n
 
-    #X = X[:n_samples]
-    #y = y[:n_samples]
+    X = X[:n_samples]
+    y = y[:n_samples]
 
-    cached_fit = mem.cache(fit_single)
+    # cached_fit = mem.cache(fit_single)
     out = Parallel(n_jobs=n_jobs, mmap_mode=None)(
-        delayed(cached_fit)(solver, X, y,
+        delayed(fit_single)(solver, X, y,
                             penalty=penalty, single_target=single_target,
                             dtype=dtype,
                             C=1, max_iter=max_iter, skip_slow=skip_slow)
@@ -197,54 +200,62 @@ def plot():
 
     grouped = res.groupby(level=['single_target', 'penalty'])
 
-    colors = {'saga': 'blue', 'liblinear': 'orange', 'lightning': 'green'}
+    colors = {'saga': 'C0', 'liblinear': 'C1', 'lightning': 'C2'}
     linestyles = {"float32": "--", "float64": "-"}
+    alpha = {"float64": 0.5, "float32": 1}
 
     for idx, group in grouped:
         single_target, penalty = idx
-        fig = plt.figure(figsize=(12, 4))
-        ax = fig.add_subplot(131)
-
-        train_scores = group['train_scores'].values
-        ref = np.min(np.concatenate(train_scores)) * 0.999
+        fig, axes = plt.subplots(figsize=(12, 4), ncols=4)
+        ax = axes[0]
 
         for scores, times, solver, dtype in zip(group['train_scores'],
                                                 group['times'],
                                                 group['solver'],
                                                 group["dtype"]):
-            scores = scores / ref - 1
             ax.plot(times, scores, label="%s - %s" % (solver, dtype),
                     color=colors[solver],
+                    alpha=alpha[dtype],
+                    marker=".",
                     linestyle=linestyles[dtype])
+            ax.axvline(times[-1], color=colors[solver],
+                       alpha=alpha[dtype],
+                       linestyle=linestyles[dtype])
         ax.set_xlabel('Time (s)')
         ax.set_ylabel('Training objective (relative to min)')
         ax.set_yscale('log')
 
-        ax = fig.add_subplot(132)
-
-        test_scores = group['test_scores'].values
-        ref = np.min(np.concatenate(test_scores)) * 0.999
+        ax = axes[1]
 
         for scores, times, solver, dtype in zip(group['test_scores'],
                                                 group['times'],
                                                 group['solver'],
                                                 group["dtype"]):
-            scores = scores / ref - 1
             ax.plot(times, scores, label=solver, color=colors[solver],
-                    linestyle=linestyles[dtype])
+                    linestyle=linestyles[dtype],
+                    marker=".",
+                    alpha=alpha[dtype])
+            ax.axvline(times[-1], color=colors[solver],
+                       alpha=alpha[dtype],
+                       linestyle=linestyles[dtype])
 
         ax.set_xlabel('Time (s)')
         ax.set_ylabel('Test objective (relative to min)')
         ax.set_yscale('log')
 
-        ax = fig.add_subplot(133)
-
+        ax = axes[2]
         for accuracy, times, solver, dtype in zip(group['accuracies'],
                                                   group['times'],
                                                   group['solver'],
                                                   group["dtype"]):
             ax.plot(times, accuracy, label="%s - %s" % (solver, dtype),
+                    alpha=alpha[dtype],
+                    marker=".",
                     color=colors[solver], linestyle=linestyles[dtype])
+            ax.axvline(times[-1], color=colors[solver],
+                       alpha=alpha[dtype],
+                       linestyle=linestyles[dtype])
+
         ax.set_xlabel('Time (s)')
         ax.set_ylabel('Test accuracy')
         ax.legend()
@@ -254,14 +265,31 @@ def plot():
         name += '.png'
         fig.tight_layout()
         fig.subplots_adjust(top=0.9)
+
+        ax = axes[3]
+        for scores, times, solver, dtype in zip(group['train_scores'],
+                                                group['times'],
+                                                group['solver'],
+                                                group["dtype"]):
+            ax.plot(np.arange(len(scores)),
+                    scores, label="%s - %s" % (solver, dtype),
+                    marker=".",
+                    alpha=alpha[dtype],
+                    color=colors[solver], linestyle=linestyles[dtype])
+
+        ax.set_yscale("log")
+        ax.set_xlabel('# iterations')
+        ax.set_ylabel('Objective function')
+        ax.legend()
+
         plt.savefig(name)
-        plt.close(fig)
 
 
 if __name__ == '__main__':
-    solvers = ['saga', 'liblinear'] #'lightning']
+    solvers = ['saga', 'liblinear']
     penalties = ['l1', 'l2']
     single_target = True
-    exp(solvers, penalties, single_target, n_samples=None, n_jobs=1,
-        dataset='rcv1', max_iter=20)
+    exp(solvers, penalties, single_target,
+        n_samples=None, n_jobs=1,
+        dataset='rcv1', max_iter=10)
     plot()
