AI Experiment Set Up
An optimization experiment leverages SigOpt's proprietary ensemble of global optimization algorithms to learn about the relationship between parameters and metrics over the experiment lifecycle. SigOpt allows you to search for a single high performing parameter configuration or a set of high performing parameter configurations with SigOpt's Multimetric Optimization.
AI Experiments can be created either in a script with calls from a SigOpt client library or by defining the AI Experiment in a YAML file that will be passed to the SigOpt CLI.
For notebook instructions and tutorials, check out our GitHub notebook tutorials repo, open the SigOpt AI Experiment notebook tutorial in Google Colab.
Here is how to execute a SigOpt AI Experiment using Python and the SigOpt CLI.
Copy and paste the code below to a file named
sigopt_bo_experiment.py:import tensorflow as tf
import sigopt
import os
os.environ["SIGOPT_API_TOKEN"] = "YOUR_API_TOKEN"
os.environ["SIGOPT_PROJECT"] = "run-examples"
class KerasNNModel:
def __init__(self, hidden_layer_size, activation_fn):
model = tf.keras.Sequential(
[
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(hidden_layer_size, activation=activation_fn),
tf.keras.layers.Dense(10),
]
)
self.model = model
def get_keras_nn_model(self):
return self.model
def train_model(self, train_images, train_labels, optimizer_type, metrics_list, num_epochs):
self.model.compile(
optimizer=optimizer_type,
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=metrics_list,
)
self.model.fit(train_images, train_labels, epochs=num_epochs)
def evaluate_model(self, test_images, test_labels):
metrics_dict = self.model.evaluate(test_images, test_labels, verbose=2, return_dict=True)
return metrics_dict
def load_data_train_model():
sigopt.log_dataset(name="mnist")
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
# set model training, architecture parameters and hyperparameters
sigopt.params.num_epochs = 2
# create the model
keras_nn_model = KerasNNModel(
hidden_layer_size=sigopt.params.hidden_layer_size, activation_fn=sigopt.params.activation_function
)
sigopt.log_model("Keras NN Model with 1 Hidden layer")
# train the model
keras_nn_model.train_model(train_images, train_labels, "adam", ["accuracy"], sigopt.params.num_epochs)
sigopt.log_metadata("sgd optimizer", "adam")
metrics_dict = keras_nn_model.evaluate_model(test_images, test_labels)
# log performance metrics
sigopt.log_metric("holdout_accuracy", metrics_dict["accuracy"])
if __name__ == "__main__":
load_data_train_model()
Copy and paste the yml file below to a file named
experiment.yml:name: Single metric optimization
type: offline
parameters:
- name: hidden_layer_size
type: int
bounds:
min: 32
max: 512
- name: activation_function
type: categorical
categorical_values:
- relu
- tanh
metrics:
- name: holdout_accuracy
strategy: optimize
objective: maximize
parallel_bandwidth: 1
budget: 30
Execute the CLI command below to start your AI Experiment:
$ sigopt optimize -e experiment.yml python sigopt_bo_experiment.py
And that’s it! Navigate to the SigOpt web application to keep an eye on your AI Experiment and draw insights from your results!
Copy and paste the code below to a file named
sigopt_bo_experiment_nocli.py:import tensorflow as tf
import sigopt
import os
os.environ["SIGOPT_API_TOKEN"] = "YOUR_API_TOKEN"
os.environ["SIGOPT_PROJECT"] = "run-examples"
class KerasNNModel:
def __init__(self, hidden_layer_size, activation_fn):
model = tf.keras.Sequential(
[
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(hidden_layer_size, activation=activation_fn),
tf.keras.layers.Dense(10),
]
)
self.model = model
def get_keras_nn_model(self):
return self.model
def train_model(self, train_images, train_labels, optimizer_type, metrics_list, num_epochs):
self.model.compile(
optimizer=optimizer_type,
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=metrics_list,
)
self.model.fit(train_images, train_labels, epochs=num_epochs)
def evaluate_model(self, test_images, test_labels):
metrics_dict = self.model.evaluate(test_images, test_labels, verbose=2, return_dict=True)
return metrics_dict
def load_data_train_model(run):
run.log_dataset(name="mnist")
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
# set model training, architecture parameters and hyperparameters
run.params.num_epochs = 2
# create the model
keras_nn_model = KerasNNModel(
hidden_layer_size=run.params.hidden_layer_size, activation_fn=run.params.activation_function
)
run.log_model("Keras NN Model with 1 Hidden layer")
# train the model
keras_nn_model.train_model(train_images, train_labels, "adam", ["accuracy"], run.params.num_epochs)
run.log_metadata("sgd optimizer", "adam")
metrics_dict = keras_nn_model.evaluate_model(test_images, test_labels)
# log performance metrics
run.log_metric("holdout_accuracy", metrics_dict["accuracy"])
if __name__ == "__main__":
experiment = sigopt.create_experiment(
name="Single metric optimization",
type="offline",
parameters=[
dict(name="hidden_layer_size", type="int", bounds=dict(min=32, max=512)),
dict(name="activation_function", type="categorical", categorical_values=["relu", "tanh"]),
],
metrics=[dict(name="holdout_accuracy", strategy="optimize", objective="maximize")],
parallel_bandwidth=1,
budget=30,
)
for run in experiment.loop():
with run:
load_data_train_model(run=run)
Execute the CLI command below to start your AI Experiment:
$ python sigopt_bo_experiment_nocli.py
And that’s it! Navigate to the SigOpt web application to keep an eye on your AI Experiment and draw insights from your results!