import copy
import json
import ConfigSpace as cs
import ConfigSpace.hyperparameters as csh
import numpy as np
from ConfigSpace.read_and_write import json as cs_json
import deephyper.core.exceptions as dh_exceptions
import deephyper.skopt
def convert_to_skopt_dim(cs_hp, surrogate_model=None):
if surrogate_model in ["RF", "ET", "GBRT"]:
# models not sensitive to the metric space such as trees
surrogate_model_type = "rule_based"
else:
# models sensitive to the metric space such as GP, neural networks
surrogate_model_type = "distance_based"
if isinstance(cs_hp, csh.UniformIntegerHyperparameter):
skopt_dim = deephyper.skopt.space.Integer(
low=cs_hp.lower,
high=cs_hp.upper,
prior="log-uniform" if cs_hp.log else "uniform",
name=cs_hp.name,
)
elif isinstance(cs_hp, csh.UniformFloatHyperparameter):
skopt_dim = deephyper.skopt.space.Real(
low=cs_hp.lower,
high=cs_hp.upper,
prior="log-uniform" if cs_hp.log else "uniform",
name=cs_hp.name,
)
elif isinstance(cs_hp, csh.CategoricalHyperparameter):
# the transform is important if we don't want the complexity of trees
# to explode with categorical variables
skopt_dim = deephyper.skopt.space.Categorical(
categories=cs_hp.choices,
name=cs_hp.name,
transform="onehot" if surrogate_model_type == "distance_based" else "label",
)
elif isinstance(cs_hp, csh.OrdinalHyperparameter):
categories = list(cs_hp.sequence)
if all(
isinstance(x, (int, np.integer)) or isinstance(x, (float, np.floating))
for x in categories
):
transform = "identity"
else:
transform = "label"
skopt_dim = deephyper.skopt.space.Categorical(
categories=categories, name=cs_hp.name, transform=transform
)
else:
raise TypeError(f"Cannot convert hyperparameter of type {type(cs_hp)}")
return skopt_dim
def convert_to_skopt_space(cs_space, surrogate_model=None):
"""Convert a ConfigurationSpace to a scikit-optimize Space.
Args:
cs_space (ConfigurationSpace): the ``ConfigurationSpace`` to convert.
surrogate_model (str, optional): the type of surrogate model/base estimator used to perform Bayesian optimization. Defaults to None.
Raises:
TypeError: if the input space is not a ConfigurationSpace.
RuntimeError: if the input space contains forbiddens.
RuntimeError: if the input space contains conditions
Returns:
deephyper.skopt.space.Space: a scikit-optimize Space.
"""
# verify pre-conditions
if not (isinstance(cs_space, cs.ConfigurationSpace)):
raise TypeError("Input space should be of type ConfigurationSpace")
if len(cs_space.get_conditions()) > 0:
raise RuntimeError("Cannot convert a ConfigSpace with Conditions!")
if len(cs_space.get_forbiddens()) > 0:
raise RuntimeError("Cannot convert a ConfigSpace with Forbiddens!")
# convert the ConfigSpace to deephyper.skopt.space.Space
dimensions = []
for hp in cs_space.get_hyperparameters():
dimensions.append(convert_to_skopt_dim(hp, surrogate_model))
skopt_space = deephyper.skopt.space.Space(dimensions)
return skopt_space
def check_hyperparameter(parameter, name=None, default_value=None):
"""Check if the passed parameter is a valid description of an hyperparameter.
:meta private:
Args:
parameter (str|Hyperparameter): an instance of ``ConfigSpace.hyperparameters.hyperparameter`` or a synthetic description (e.g., ``list``, ``tuple``).
parameter (str): the name of the hyperparameter. Only required when the parameter is not a ``ConfigSpace.hyperparameters.hyperparameter``.
default_value: a default value for the hyperparameter.
Returns:
Hyperparameter: the ConfigSpace hyperparameter instance corresponding to the ``parameter`` description.
"""
if isinstance(parameter, csh.Hyperparameter):
return parameter
if not isinstance(parameter, (list, tuple, np.ndarray, dict)):
raise ValueError(
"Shortcut definition of an hyper-parameter has to be a list, tuple, array or dict."
)
if not (type(name) is str):
raise ValueError("The 'name' of an hyper-parameter should be a string!")
kwargs = {}
if default_value is not None:
kwargs["default_value"] = default_value
if type(parameter) is tuple: # Range of reals or integers
if len(parameter) == 2:
prior = "uniform"
elif len(parameter) == 3:
prior = parameter[2]
assert prior in [
"uniform",
"log-uniform",
], f"Prior has to be 'uniform' or 'log-uniform' when {prior} was given for parameter '{name}'"
parameter = parameter[:2]
log = prior == "log-uniform"
if all([isinstance(p, int) for p in parameter]):
return csh.UniformIntegerHyperparameter(
name=name, lower=parameter[0], upper=parameter[1], log=log, **kwargs
)
elif any([isinstance(p, float) for p in parameter]):
return csh.UniformFloatHyperparameter(
name=name, lower=parameter[0], upper=parameter[1], log=log, **kwargs
)
elif type(parameter) is list: # Categorical
if any(
[isinstance(p, (str, bool)) or isinstance(p, np.bool_) for p in parameter]
):
return csh.CategoricalHyperparameter(name, choices=parameter, **kwargs)
elif all([isinstance(p, (int, float)) for p in parameter]):
return csh.OrdinalHyperparameter(name, sequence=parameter, **kwargs)
elif type(parameter) is dict: # Integer or Real distribution
# Normal
if "mu" in parameter and "sigma" in parameter:
if type(parameter["mu"]) is float:
return csh.NormalFloatHyperparameter(name=name, **parameter, **kwargs)
elif type(parameter["mu"]) is int:
return csh.NormalIntegerHyperparameter(name=name, **parameter, **kwargs)
else:
raise ValueError(
"Wrong hyperparameter definition! 'mu' should be either a float or an integer."
)
raise ValueError(
f"Invalid dimension {name}: {parameter}. Read the documentation for"
f" supported types."
)
[docs]class HpProblem:
"""Class to define an hyperparameter problem.
>>> from deephyper.problem import HpProblem
>>> problem = HpProblem()
Args:
config_space (ConfigurationSpace, optional): In case the ``HpProblem`` is defined from a `ConfigurationSpace`.
"""
def __init__(self, config_space=None):
if config_space is not None and not (
isinstance(config_space, cs.ConfigurationSpace)
):
raise ValueError(
"Parameter 'config_space' should be an instance of ConfigurationSpace!"
)
if config_space:
self._space = copy.deepcopy(config_space)
else:
self._space = cs.ConfigurationSpace()
self.references = [] # starting points
def __str__(self):
return repr(self)
def __repr__(self):
prob = repr(self._space)
return prob
[docs] def add_hyperparameter(
self, value, name: str = None, default_value=None
) -> csh.Hyperparameter:
"""Add an hyperparameter to the ``HpProblem``.
Hyperparameters can be added to a ``HpProblem`` with a short syntax:
>>> problem.add_hyperparameter((0, 10), "discrete", default_value=5)
>>> problem.add_hyperparameter((0.0, 10.0), "real", default_value=5.0)
>>> problem.add_hyperparameter([0, 10], "categorical", default_value=0)
Sampling distributions can be provided:
>>> problem.add_hyperparameter((0.0, 10.0, "log-uniform"), "real", default_value=5.0)
It is also possible to use `ConfigSpace <https://automl.github.io/ConfigSpace/master/API-Doc.html#hyperparameters>`_ ``Hyperparameters``:
>>> import ConfigSpace.hyperparameters as csh
>>> csh_hp = csh.UniformIntegerHyperparameter(
... name='uni_int', lower=10, upper=100, log=False)
>>> problem.add_hyperparameter(csh_hp)
Args:
value (tuple or list or ConfigSpace.Hyperparameter): a valid hyperparametr description.
name (str): The name of the hyperparameter to add.
default_value (float or int or str): A default value for the corresponding hyperparameter.
Returns:
ConfigSpace.Hyperparameter: a ConfigSpace ``Hyperparameter`` object corresponding to the ``(value, name, default_value)``.
"""
if not (type(name) is str or name is None):
raise dh_exceptions.problem.SpaceDimNameOfWrongType(name)
csh_parameter = check_hyperparameter(value, name, default_value=default_value)
self._space.add_hyperparameter(csh_parameter)
return csh_parameter
[docs] def add_hyperparameters(self, hp_list):
"""Add a list of hyperparameters. It can be useful when a list of ``ConfigSpace.Hyperparameter`` are defined and we need to add them to the ``HpProblem``.
Args:
hp_list (ConfigSpace.Hyperparameter): a list of ConfigSpace hyperparameters.
Returns:
list: The list of added hyperparameters.
"""
return [self.add_hyperparameter(hp) for hp in hp_list]
[docs] def add_forbidden_clause(self, clause):
"""Add a `forbidden clause <https://automl.github.io/ConfigSpace/master/API-Doc.html#forbidden-clauses>`_ to the ``HpProblem``.
For example if we want to optimize :math:`\\frac{1}{x}` where :math:`x` cannot be equal to 0:
>>> from deephyper.problem import HpProblem
>>> import ConfigSpace as cs
>>> problem = HpProblem()
>>> x = problem.add_hyperparameter((0.0, 10.0), "x")
>>> problem.add_forbidden_clause(cs.ForbiddenEqualsClause(x, 0.0))
Args:
clause: a ConfigSpace forbidden clause.
"""
self._space.add_forbidden_clause(clause)
[docs] def add_condition(self, condition):
"""Add a `condition <https://automl.github.io/ConfigSpace/master/API-Doc.html#conditions>`_ to the ``HpProblem``.
>>> from deephyper.problem import HpProblem
>>> import ConfigSpace as cs
>>> problem = HpProblem()
>>> x = problem.add_hyperparameter((0.0, 10.0), "x")
>>> y = problem.add_hyperparameter((1e-4, 1.0), "y")
>>> problem.add_condition(cs.LessThanCondition(y, x, 1.0))
Args:
condition: A ConfigSpace condition.
"""
self._space.add_condition(condition)
[docs] def add_conditions(self, conditions: list) -> None:
"""Add a list of `condition <https://automl.github.io/ConfigSpace/master/API-Doc.html#conditions>`_ to the ``HpProblem``.
Args:
conditions (list): A list of ConfigSpace conditions.
"""
self._space.add_conditions(conditions)
@property
def space(self):
"""The wrapped ConfigSpace object."""
return self._space
@property
def hyperparameter_names(self):
"""The list of hyperparameters names."""
return self._space.get_hyperparameter_names()
[docs] def check_configuration(self, parameters: dict):
"""Check if a configuration is valid. Raise an error if not."""
config = cs.Configuration(self._space, parameters)
self._space.check_configuration(config)
@property
def default_configuration(self):
"""The default configuration as a dictionnary."""
config = self._space.get_default_configuration().get_dictionary()
return config
[docs] def to_json(self):
"""Returns a dict version of the space which can be saved as JSON."""
json_format = json.loads(cs_json.write(self._space))
return json_format