Note
Go to the end to download the full example code.
Profile the Worker Utilization#
Author(s): Romain Egele.
This example demonstrates the advantages of parallel evaluations over serial evaluations. We start by defining an artificial black-box run-function by using the Ackley function:
We will use the time.sleep function to simulate a budget of 2 secondes of execution in average which helps illustrate the advantage of parallel evaluations. The @profile decorator is useful to collect starting/ending time of the run-function execution which help us know exactly when we are inside the black-box. This decorator is necessary when profiling the worker utilization. When using this decorator, the run-function will return a dictionnary with 2 new keys "timestamp_start" and "timestamp_end". The run-function is defined in a separate module because of the “multiprocessing” backend that we are using in this example.
1"""Set of Black-Box functions useful to build examples.
2"""
3import time
4import numpy as np
5from deephyper.evaluator import profile
6
7
8def ackley(x, a=20, b=0.2, c=2 * np.pi):
9 d = len(x)
10 s1 = np.sum(x**2)
11 s2 = np.sum(np.cos(c * x))
12 term1 = -a * np.exp(-b * np.sqrt(s1 / d))
13 term2 = -np.exp(s2 / d)
14 y = term1 + term2 + a + np.exp(1)
15 return y
16
17
18@profile
19def run_ackley(config, sleep_loc=2, sleep_scale=0.5):
20
21 # to simulate the computation of an expensive black-box
22 if sleep_loc > 0:
23 t_sleep = np.random.normal(loc=sleep_loc, scale=sleep_scale)
24 t_sleep = max(t_sleep, 0)
25 time.sleep(t_sleep)
26
27 x = np.array([config[k] for k in config if "x" in k])
28 x = np.asarray_chkfinite(x) # ValueError if any NaN or Inf
29 return -ackley(x) # maximisation is performed
After defining the black-box we can continue with the definition of our main script:
import black_box_util as black_box
from deephyper.analysis._matplotlib import update_matplotlib_rc
update_matplotlib_rc()
Then we define the variable(s) we want to optimize. For this problem we optimize Ackley in a 2-dimensional search space, the true minimul is located at (0, 0).
Configuration space object:
Hyperparameters:
x0, Type: UniformFloat, Range: [-32.768, 32.768], Default: 0.0
x1, Type: UniformFloat, Range: [-32.768, 32.768], Default: 0.0
Then we define a parallel search.
if __name__ == "__main__":
from deephyper.evaluator import Evaluator
from deephyper.evaluator.callback import TqdmCallback
from deephyper.hpo import CBO
timeout = 20
num_workers = 4
results = {}
evaluator = Evaluator.create(
black_box.run_ackley,
method="process",
method_kwargs={
"num_workers": num_workers,
"callbacks": [TqdmCallback()],
},
)
search = CBO(problem, evaluator, random_state=42)
results = search.search(timeout=timeout)
0it [00:00, ?it/s]
1it [00:00, 2976.79it/s, failures=0, objective=-20.2]
2it [00:01, 1.60it/s, failures=0, objective=-20.2]
2it [00:01, 1.60it/s, failures=0, objective=-19.8]
3it [00:01, 1.89it/s, failures=0, objective=-19.8]
3it [00:01, 1.89it/s, failures=0, objective=-19.8]
4it [00:01, 2.27it/s, failures=0, objective=-19.8]
4it [00:01, 2.27it/s, failures=0, objective=-19.8]
5it [00:02, 2.45it/s, failures=0, objective=-19.8]
5it [00:02, 2.45it/s, failures=0, objective=-19.8]
6it [00:03, 1.36it/s, failures=0, objective=-19.8]
6it [00:03, 1.36it/s, failures=0, objective=-19.8]
7it [00:03, 1.83it/s, failures=0, objective=-19.8]
7it [00:03, 1.83it/s, failures=0, objective=-19.8]
8it [00:04, 2.31it/s, failures=0, objective=-19.8]
8it [00:04, 2.31it/s, failures=0, objective=-15.4]
9it [00:04, 2.94it/s, failures=0, objective=-15.4]
9it [00:04, 2.94it/s, failures=0, objective=-15.4]
10it [00:05, 1.71it/s, failures=0, objective=-15.4]
10it [00:05, 1.71it/s, failures=0, objective=-15.4]
11it [00:05, 1.70it/s, failures=0, objective=-15.4]
11it [00:05, 1.70it/s, failures=0, objective=-15.4]
12it [00:06, 1.70it/s, failures=0, objective=-15.4]
12it [00:06, 1.70it/s, failures=0, objective=-15.4]
13it [00:06, 2.20it/s, failures=0, objective=-15.4]
13it [00:06, 2.20it/s, failures=0, objective=-12.6]
14it [00:07, 1.79it/s, failures=0, objective=-12.6]
14it [00:07, 1.79it/s, failures=0, objective=-12.6]
15it [00:07, 2.25it/s, failures=0, objective=-12.6]
15it [00:07, 2.25it/s, failures=0, objective=-12.6]
16it [00:08, 1.68it/s, failures=0, objective=-12.6]
16it [00:08, 1.68it/s, failures=0, objective=-12.6]
17it [00:09, 1.62it/s, failures=0, objective=-12.6]
17it [00:09, 1.62it/s, failures=0, objective=-12.6]
18it [00:09, 2.01it/s, failures=0, objective=-12.6]
18it [00:09, 2.01it/s, failures=0, objective=-6.84]
19it [00:09, 2.58it/s, failures=0, objective=-6.84]
19it [00:09, 2.58it/s, failures=0, objective=-6.37]
20it [00:10, 1.62it/s, failures=0, objective=-6.37]
20it [00:10, 1.62it/s, failures=0, objective=-5.35]
21it [00:11, 1.33it/s, failures=0, objective=-5.35]
21it [00:11, 1.33it/s, failures=0, objective=-5.35]
22it [00:12, 1.58it/s, failures=0, objective=-5.35]
22it [00:12, 1.58it/s, failures=0, objective=-5.35]
23it [00:12, 1.65it/s, failures=0, objective=-5.35]
23it [00:12, 1.65it/s, failures=0, objective=-5.35]
24it [00:12, 2.16it/s, failures=0, objective=-5.35]
24it [00:12, 2.16it/s, failures=0, objective=-5.35]
25it [00:13, 2.06it/s, failures=0, objective=-5.35]
25it [00:13, 2.06it/s, failures=0, objective=-5.35]
26it [00:14, 1.77it/s, failures=0, objective=-5.35]
26it [00:14, 1.77it/s, failures=0, objective=-5.35]
27it [00:14, 2.21it/s, failures=0, objective=-5.35]
27it [00:14, 2.21it/s, failures=0, objective=-5.35]
28it [00:14, 2.18it/s, failures=0, objective=-5.35]
28it [00:14, 2.18it/s, failures=0, objective=-5.35]
29it [00:15, 2.40it/s, failures=0, objective=-5.35]
29it [00:15, 2.40it/s, failures=0, objective=-5.35]
30it [00:16, 1.26it/s, failures=0, objective=-5.35]
30it [00:16, 1.26it/s, failures=0, objective=-5.35]
31it [00:16, 1.67it/s, failures=0, objective=-5.35]
31it [00:16, 1.67it/s, failures=0, objective=-5.35]
32it [00:17, 2.18it/s, failures=0, objective=-5.35]
32it [00:17, 2.18it/s, failures=0, objective=-5.35]
Finally, we plot the results from the collected DataFrame.
if __name__ == "__main__":
import matplotlib.pyplot as plt
import numpy as np
def compile_profile(df):
"""Take the results dataframe as input and return the number of jobs running at a given timestamp."""
history = []
for _, row in df.iterrows():
history.append((row["m:timestamp_start"], 1))
history.append((row["m:timestamp_end"], -1))
history = sorted(history, key=lambda v: v[0])
nb_workers = 0
timestamp = [0]
n_jobs_running = [0]
for time, incr in history:
nb_workers += incr
timestamp.append(time)
n_jobs_running.append(nb_workers)
return timestamp, n_jobs_running
t0 = results["m:timestamp_start"].iloc[0]
results["m:timestamp_start"] = results["m:timestamp_start"] - t0
results["m:timestamp_end"] = results["m:timestamp_end"] - t0
tmax = results["m:timestamp_end"].max()
plt.figure()
plt.subplot(2, 1, 1)
plt.scatter(results["m:timestamp_end"], results.objective)
plt.plot(results["m:timestamp_end"], results.objective.cummax())
plt.xlabel("Time (sec.)")
plt.ylabel("Objective")
plt.grid()
plt.xlim(0, tmax)
plt.subplot(2, 1, 2)
x, y = compile_profile(results)
y = np.asarray(y) / num_workers * 100
plt.step(
x,
y,
where="pre",
)
plt.ylim(0, 100)
plt.xlim(0, tmax)
plt.xlabel("Time (sec.)")
plt.ylabel("Worker Utilization (\\%)")
plt.tight_layout()
plt.show()
Total running time of the script: (0 minutes 20.473 seconds)