Record SigOpt Runs with Python IDE
If the CLI or Jupyter Notebook integration isn't right for your use case then you might want to create Runs and Experiments directly with the SigOpt Python Client. To create a Run, just add the following to your code:
1
run = sigopt.create_run()
Copied!
With this Run object you can get and set parameter values and log attributes in a similar way as you would when using the CLI:
1
run.params.learning_rate = 0.1
2
accuracy = train_my_model(learning_rate=run.params.learning_rate)
3
run.log_metric("accuracy", accuracy)
Copied!
Finally, end the Run:
1
run.end()
Copied!
For convenience, you can use a Python context manager to end the Run automatically, including when your code raises an exception:
1
with sigopt.create_run() as run:
2
run.params.learning_rate = 0.1
3
accuracy = train_my_model(learning_rate=sigopt.params.learning_rate)
4
run.log_metric("accuracy", accuracy)
Copied!
With SigOpt installed and your Python environment set up, let's take a look at how to record a SigOpt Run in a Python IDE.

Instrument the Run

1
import tensorflow as tf
2
import sigopt
3
import os
4
5
os.environ["SIGOPT_API_TOKEN"] = # SIGOPT-API-TOKEN
6
os.environ["SIGOPT_PROJECT"] = "run-examples"
7
8
9
class KerasNNModel:
10
def __init__(self, hidden_layer_size, activation_fn):
11
model = tf.keras.Sequential(
12
[
13
tf.keras.layers.Flatten(input_shape=(28, 28)),
14
tf.keras.layers.Dense(hidden_layer_size, activation=activation_fn),
15
tf.keras.layers.Dense(10),
16
]
17
)
18
self.model = model
19
20
def get_keras_nn_model(self):
21
return self.model
22
23
def train_model(self, train_images, train_labels, optimizer_type, metrics_list, num_epochs):
24
self.model.compile(
25
optimizer=optimizer_type,
26
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
27
metrics=metrics_list,
28
)
29
self.model.fit(train_images, train_labels, epochs=num_epochs)
30
31
def evaluate_model(self, test_images, test_labels):
32
metrics_dict = self.model.evaluate(test_images, test_labels, verbose=2, return_dict=True)
33
return metrics_dict
34
35
36
def load_data_train_model(sigopt_run):
37
sigopt_run.log_dataset(name="mnist")
38
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
39
40
# set model training, architecture parameters and hyperparameters
41
sigopt_run.params.num_epochs = 2
42
sigopt_run.params.hidden_layer_size = 200
43
sigopt_run.params.activation_fn = "relu"
44
45
# create the model
46
keras_nn_model = KerasNNModel(
47
hidden_layer_size=sigopt_run.params.hidden_layer_size, activation_fn=sigopt_run.params.activation_fn
48
)
49
sigopt_run.log_model("Keras NN Model with 1 Hidden layer")
50
51
# train the model
52
keras_nn_model.train_model(train_images, train_labels, "adam", ["accuracy"], sigopt_run.params.num_epochs)
53
sigopt_run.log_metadata("sgd optimizer", "adam")
54
metrics_dict = keras_nn_model.evaluate_model(test_images, test_labels)
55
56
# log performance metrics
57
sigopt_run.log_metric("accuracy", metrics_dict["accuracy"])
58
sigopt_run.log_metric("loss", metrics_dict["loss"])
59
60
61
if __name__ == "__main__":
62
with sigopt.create_run() as run:
63
load_data_train_model(sigopt_run=run)pyth
Copied!

Run the Code

1
$ python keras_model.py
Copied!
1
Run started, view it on the SigOpt dashboard at https://app.sigopt.com/run/1234
2
Epoch 1/2
3
1875/1875 [==============================] - 5s 2ms/step - loss: 2.7513 - accuracy: 0.8826
4
Epoch 2/2
5
1875/1875 [==============================] - 4s 2ms/step - loss: 0.3313 - accuracy: 0.9265
6
313/313 - 0s - loss: 0.2941 - accuracy: 0.9478
7
Run finished, view it on the SigOpt dashboard at https://app.sigopt.com/run/1234
Copied!