Add performance test for LexiFlow (#812)

* add test

* fix

* change test name

flaml/tune/searcher/flow2.py

@@ -358,25 +358,23 @@ def set_search_properties(
  def update_fbest(
  self,
  ):
- # TODO: Improve the efficiency
  obj_initial = self.lexico_objectives["metrics"][0]
- feasible_index = [*range(len(self._histories[obj_initial]))]
+ feasible_index = np.array([*range(len(self._histories[obj_initial]))])
  for k_metric in self.lexico_objectives["metrics"]:
  k_values = np.array(self._histories[k_metric])
- self._f_best[k_metric] = np.min(k_values.take(feasible_index))
- feasible_index_prior = np.where(
- k_values
+ feasible_value = k_values.take(feasible_index)
+ self._f_best[k_metric] = np.min(feasible_value)
+ feasible_index_filter = np.where(
+ feasible_value
  <= max(
  [
  self._f_best[k_metric]
  + self.lexico_objectives["tolerances"][k_metric],
  self.lexico_objectives["targets"][k_metric],
  ]
  )
- )[0].tolist()
- feasible_index = [
- val for val in feasible_index if val in feasible_index_prior
- ]
+ )[0]
+ feasible_index = feasible_index.take(feasible_index_filter)
 
  def lexico_compare(self, result) -> bool:
  if self._histories is None:

flaml/tune/tune.py

@@ -74,10 +74,10 @@ def lexico_best(self, trials):
  histories[objective].append(
  results[keys[time_index]][objective]
  if mode == "min"
- else -trials[keys[time_index]][objective]
+ else -results[keys[time_index]][objective]
  )
  obj_initial = self.lexico_objectives["metrics"][0]
- feasible_index = [*range(len(histories[obj_initial]))]
+ feasible_index = np.array([*range(len(histories[obj_initial]))])
  for k_metric, k_mode in zip(
  self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
  ):
@@ -87,20 +87,19 @@ def lexico_best(self, trials):
  if k_mode == "max"
  else self.lexico_objectives["targets"][k_metric]
  )
- f_best[k_metric] = np.min(k_values.take(feasible_index))
- feasible_index_prior = np.where(
- k_values
+ feasible_value = k_values.take(feasible_index)
+ f_best[k_metric] = np.min(feasible_value)
+ feasible_index_filter = np.where(
+ feasible_value
  <= max(
  [
  f_best[k_metric]
  + self.lexico_objectives["tolerances"][k_metric],
  k_target,
  ]
  )
- )[0].tolist()
- feasible_index = [
- val for val in feasible_index if val in feasible_index_prior
- ]
+ )[0]
+ feasible_index = feasible_index.take(feasible_index_filter)
  best_trial = trials[feasible_index[-1]]
  return best_trial
 

test/tune/test_lexiflow.py

@@ -1,9 +1,11 @@
 import torch
 import thop
 import torch.nn as nn
-from flaml import tune
 import torch.nn.functional as F
 import torchvision
+from flaml import tune
+from collections import defaultdict
+import math
 import numpy as np
 
 DEVICE = torch.device("cpu")
@@ -12,6 +14,24 @@
 N_VALID_EXAMPLES = BATCHSIZE * 10
 
 
+def _BraninCurrin(config):
+ # Rescale brain
+ x_1 = 15 * config["x1"] - 5
+ x_2 = 15 * config["x2"]
+ # Brain function
+ t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
+ t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
+ brain_result = t1**2 + t2 + 10
+ # Currin function
+ xc_1 = config["x1"]
+ xc_2 = config["x2"]
+ factor1 = 1 - math.exp(-1 / (2 * xc_2))
+ numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
+ denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
+ currin_result = factor1 * numer / denom
+ return {"brain": brain_result, "currin": currin_result}
+
+
 def test_lexiflow():
  train_dataset = torchvision.datasets.FashionMNIST(
  "test/data",
@@ -138,5 +158,35 @@ def evaluate_function(configuration):
  print(analysis.best_result)
 
 
+def test_lexiflow_performance():
+ lexico_objectives = {}
+ lexico_objectives["metrics"] = ["brain", "currin"]
+ lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
+ lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
+ lexico_objectives["modes"] = ["min", "min"]
+
+ search_space = {
+ "x1": tune.uniform(lower=0.000001, upper=1.0),
+ "x2": tune.uniform(lower=0.000001, upper=1.0),
+ }
+
+ analysis = tune.run(
+ _BraninCurrin,
+ num_samples=1000,
+ config=search_space,
+ use_ray=False,
+ lexico_objectives=lexico_objectives,
+ )
+
+ print(analysis.best_trial)
+ print(analysis.best_config)
+ print(analysis.best_result)
+
+ assert (
+ analysis.best_result["currin"] <= 2.2
+ ), "the value of currin function should be less than 2.2"
+
+
 if __name__ == "__main__":
  test_lexiflow()
+ test_lexiflow_performance()