Alternative Experiment Types

Random Search

A random search experiment enables you to randomly search over your parameter space with the SigOpt Platform. To create a random search Experiment, simply set the Experiment type to "random".

By default, SigOpt random suggests uniformly from the parameter space. Random search can be combined with Prior Beliefs to modify the search distribution.

Core Module

Python

experiment = conn.experiments().create(
  name="Random search",
  type="random",
  parameters=[
    dict(
      name="omp_thread",
      type="int",
      bounds=dict(
        min=1,
        max=16,
      )
    ),
    dict(
      name="omp_places",
      type="categorical",
      categorical_values=[
        "threads",
        "core",
        "sockets",
      ]
    )
  ],
  metrics=[
    dict(
      name="throughput",
      objective="maximize"
    )
  ],
  observation_budget=30
)

Java

import com.sigopt.SigOpt;
import com.sigopt.exception.SigoptException;
import com.sigopt.model.*;

import java.util.Arrays;

public class YourSigoptExperiment {
  public static Experiment createExperiment() throws SigoptException {
    Experiment experiment = Experiment.create()
      .data(
        new Experiment.Builder()
          .name("Random search")
          .parameters(java.util.Arrays.asList(
            new Parameter.Builder()
              .name("omp_places")
              .categoricalValues(java.util.Arrays.asList(
                new CategoricalValue(
                  "threads"
                  1
                  ),
                new CategoricalValue(
                  "core"
                  2
                  ),
                new CategoricalValue(
                  "sockets"
                  3
                  )
                ))
              .type("categorical")
              .build(),
            new Parameter.Builder()
              .name("omp_thread")
              .bounds(new Bounds.Builder()
                .min(1)
                .max(16)
                .build())
              .type("int")
              .build()
            ))
          .metrics(java.util.Arrays.asList(
            new Metric.Builder()
              .name("throughput")
              .objective("maximize")
              .strategy("optimize")
              .build()
            ))
          .observationBudget(30)
          .type("random")
          .build()
        )
      .call();
    return experiment;
  }

AI Module

Python

sigopt.create_experiment(
  name="Random search",
  type="random",
  parameters=[
    dict(
      name="learning_rate",
      type="double",
      bounds=dict(
        min=1e-5,
        max=1e-2
      ),
      transformation="log"
    ),
    dict(
      name="activation_function",
      type="categorical",
      categorical_values=[
        "relu",
        "tanh"
      ]
    )
  ],
  metrics=[
    dict(
      name="holdout_accuracy",
      objective="maximize"
    )
  ],
  budget=30
)

YAML

name: Random search
type: random
parameters:
  - name: learning_rate
    type: double
    bounds:
      min: 1e-5
      max: 1e-2
    transformation: log
  - name: activation_function
    type: categorical
    categorical_values:
      - relu
      - tanh
metrics:
  - name: holdout_accuracy
    objective: maximize
budget: 30

Grid Search

A grid search experiment enables users to execute an exhaustive search on a user-defined grid with the SigOpt Platform. This means that every parameter needs to have discrete values; meaning either categorical type or parameters with assigned grid values. To create a grid search Experiment, simply set the Experiment type to "grid".

Grid Search Experiments must have budget that match up with the total number of possible combination of parameter values.

Core Module

Python

experiment = conn.experiments().create(
  name="Grid search",
  type="grid",
  parameters=[
    dict(
      name="omp_thread",
      type="int",
      grid=[1, 2, 3, 4, 5, 6, 7, 8]
    ),
    dict(
      name="omp_places",
      type="categorical",
      categorical_values=[
        "threads",
        "core",
        "sockets",
      ]
    )
  ],
  metrics=[
    dict(
      name="throughput",
      objective="maximize"
    )
  ],
  observation_budget=24
)

Java

import com.sigopt.SigOpt;
import com.sigopt.exception.SigoptException;
import com.sigopt.model.*;

import java.util.Arrays;

public class YourSigoptExperiment {
  public static Experiment createExperiment() throws SigoptException {
    Experiment experiment = Experiment.create()
      .data(
        new Experiment.Builder()
          .name("Grid search")
          .parameters(java.util.Arrays.asList(
            new Parameter.Builder()
              .name("omp_places")
              .categoricalValues(java.util.Arrays.asList(
                new CategoricalValue(
                  "threads"
                  1
                  ),
                new CategoricalValue(
                  "core"
                  2
                  ),
                new CategoricalValue(
                  "sockets"
                  3
                  )
                ))
              .type("categorical")
              .build(),
            new Parameter.Builder()
              .name("omp_thread")
              .grid(java.util.Arrays.asList(
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8
                ))
              .type("int")
              .build()
            ))
          .metrics(java.util.Arrays.asList(
            new Metric.Builder()
              .name("throughput")
              .objective("maximize")
              .strategy("optimize")
              .build()
            ))
          .observationBudget(24)
          .type("grid")
          .build()
        )
      .call();
    return experiment;
  }

AI Module

Python

sigopt.create_experiment(
  name="Grid search",
  type="grid",
  parameters=[
    dict(
      name="learning_rate",
      type="double",
      grid=[1e-5, 1e-4, 1e-3, 1e-2],
      transformation="log"
    ),
    dict(
      name="kernel_size",
      type="int",
      grid=[2, 4, 6, 12]
    ),
    dict(
      name="activation_function",
      type="categorical",
      categorical_values=[
        "relu",
        "tanh"
      ]
    )
  ],
  metrics=[
    dict(
      name="holdout_accuracy",
      objective="maximize"
    )
  ],
  budget=32
)

YAML

name: Grid search
type: grid
parameters:
  - name: learning_rate
    type: double
    grid:
      - 1.0e-05
      - 0.0001
      - 0.001
      - 0.01
    transformation: log
  - name: kernel_size
    type: int
    grid:
      - 2
      - 4
      - 6
      - 12
  - name: activation_function
    type: categorical
    categorical_values:
      - relu
      - tanh
metrics:
  - name: holdout_accuracy
    objective: maximize
budget: 32