Save and load the eliobj object

Save and load the eliobj object#

Save an unfitted eliobj object (e.g., for sharing with collaborators)#

Step 0: Load necessary libraries and functions/classes#

import tensorflow_probability as tfp
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import pandas as pd
import elicit as el

from elicit.extras import utils

tfd = tfp.distributions

# numeric, standardized predictor
def std_predictor(N, quantiles):
    X = tf.cast(np.arange(N), tf.float32)
    X_std = (X-tf.reduce_mean(X))/tf.math.reduce_std(X)
    X_sel = tfp.stats.percentile(X_std, quantiles)
    return X_sel

# implemented, generative model
class ToyModel:
    def __call__(self, prior_samples, design_matrix, **kwargs):
        B = prior_samples.shape[0]
        S = prior_samples.shape[1]

        # preprocess shape of design matrix
        X = tf.broadcast_to(design_matrix[None, None,:],
                           (B,S,len(design_matrix)))
        # linear predictor (= mu)
        epred = tf.add(prior_samples[:, :, 0][:,:,None],
                       tf.multiply(prior_samples[:, :, 1][:,:,None], X)
                       )
        # data-generating model
        likelihood = tfd.Normal(
            loc=epred, scale=tf.expand_dims(prior_samples[:, :, -1], -1)
        )
        # prior predictive distribution (=height)
        ypred = likelihood.sample()
        
        # selected observations
        y_X0, y_X1, y_X2 = (ypred[:,:,0], ypred[:,:,1], ypred[:,:,2])

        return dict(
            likelihood=likelihood,
            ypred=ypred, epred=epred,
            prior_samples=prior_samples,
            y_X0=y_X0, y_X1=y_X1, y_X2=y_X2
        )

Step 1: Create the eliobj object#

# define the generative model
model=el.model(
        obj=ToyModel,
        design_matrix=std_predictor(N=200, quantiles=[25,50,75])
        )

# specify the model parameters and their prior distribution families
parameters=[
        el.parameter(
            name="beta0",
            family=tfd.Normal,
            hyperparams=dict(
                loc=el.hyper("mu0"),
                scale=el.hyper("sigma0", lower=0)
                )
        ),
        el.parameter(
            name="beta1",
            family=tfd.Normal,
            hyperparams=dict(
                loc=el.hyper("mu1"),
                scale=el.hyper("sigma1", lower=0) # TODO specify error message
                )
        ),
        el.parameter(
            name="sigma",
            family=tfd.HalfNormal,
            hyperparams=dict(
                scale=el.hyper("sigma2", lower=0)
                )
        ),
    ]

# specify the target quantities and corresponding elicitation technique
targets=[
        el.target(
            name=f"y_X{i}",
            query=el.queries.quantiles((.05, .25, .50, .75, .95)),
            loss=el.losses.MMD2(kernel="energy"),
            weight=1.0
        ) for i in range(3)
    ]

# use an oracle to simulate a ground truth for the expert data
expert=el.expert.simulator(
        ground_truth = {
            "beta0": tfd.Normal(loc=5, scale=1),
            "beta1": tfd.Normal(loc=2, scale=1),
            "sigma": tfd.HalfNormal(scale=10.0),
        },
        num_samples = 10_000
    )

# specify the optimizer for gradient descent
optimizer=el.optimizer(
        optimizer=tf.keras.optimizers.Adam,
        learning_rate=0.1,
        clipnorm=1.0
        )

# define the trainer model with used approach, seed, etc.
trainer=el.trainer(
        method="parametric_prior",
        seed=0,
        epochs=4
    )

# specify the initialization distribution, used to draw the initial values 
# for the hyperparameters
initializer=el.initializer(
        method="sobol",
        loss_quantile=0,
        iterations=8,
        distribution=el.initialization.uniform(
            radius=1,
            mean=0
            )
        )

eliobj = el.Elicit(
    model=model,
    parameters=parameters,
    targets=targets,
    expert=expert,
    optimizer=optimizer,
    trainer=trainer,
    initializer=initializer
)

Step 2: Save the unfitted eliobj object#

Two approaches are possible:

  • automatic saving: name has to be specified. The results are then saved according to the following rule: res/{method}/{name}_{seed}.pkl

  • user-specific path: file has to be specified. The path can be freely specified by the user.

# use automatic saving approach
eliobj.save(name="m1")

saved in: ./results/parametric_prior/m1_0.pkl

# use user-specific file location
eliobj.save(file="res/m1")

saved in: ./res/m1.pkl

Load and fit the unfitted eliobj object#

Load the eliobj object#

eliobj_m1 = el.utils.load("res/m1")

Fit the loaded eliobj object#

eliobj_m1.fit()

Initialization

100%|██████████| 8/8 [00:01<00:00,  7.69it/s]

 
Training

100%|██████████| 4/4 [00:01<00:00,  3.47it/s]

Inspect the fitted eliobj object#

  • results saved for each epoch are stored in history (type: dictionary)

# information saved in the history object
print(eliobj_m1.history.keys())

dict_keys(['loss', 'loss_component', 'time', 'hyperparameter', 'hyperparameter_gradient'])

  • results saved only for the last epoch are stored in results (type: dictionary)

# information saved in the results object
print(eliobj_m1.results.keys())

dict_keys(['target_quantities', 'elicited_statistics', 'prior_samples', 'model_samples', 'model', 'loss_tensor_expert', 'loss_tensor_model', 'expert_elicited_statistics', 'expert_prior_samples', 'init_loss_list', 'init_prior', 'init_matrix'])

Save only subset of results#

Sometimes you don’t want to save all possible results but only a relevant subset. You can control this by the arguments save_configs_history and save_configs_results in the el.trainer callable.

Example

I don’t want to save information about the hyperparameter gradients and the single loss components. This can be done as follows:

import copy

# copy eliobj_m1 (to keep the new and orig. object in environment)
eliobj_m2 = copy.deepcopy(eliobj_m1)

# update eliobj_m1 by changing the saving settings
trainer_new = el.trainer(
        method="parametric_prior",
        seed=0,
        epochs=4
    )
eliobj_m2.update(trainer=trainer_new)

# fit updated eliobj
eliobj_m2.fit(save_history=el.utils.save_history(hyperparameter_gradient=False, loss_component=False))

# inspect saved results
# note that loss_component and hyperparameter_gradient are not saved
eliobj_m2.history.keys()

C:\Users\bockting\Documents\GitHub\prior_elicitation\elicit\utils.py:569: UserWarning: el.plots.loss() requires information about 'loss' and 'loss_component'. If you don't save this information el.plot.loss() can't be used.
  warnings.warn(

INFO: Results have been reset.
Initialization

100%|██████████| 8/8 [00:01<00:00,  7.44it/s]

 
Training

100%|██████████| 4/4 [00:00<00:00,  4.18it/s]

dict_keys(['loss', 'time', 'hyperparameter'])

Save and reload the fitted eliobj object#

Step 1: Save the fitted object#

eliobj_m2.save(name="m2")

saved in: ./results/parametric_prior/m2_0.pkl

Step 2: Load and inspect the fitted eliobj object#

eliobj_m2_reload = el.utils.load("res/parametric_prior/m2_0")
eliobj_m2_reload.history["loss"]

[<tf.Tensor: shape=(), dtype=float32, numpy=14.244483>,
 <tf.Tensor: shape=(), dtype=float32, numpy=13.580679>,
 <tf.Tensor: shape=(), dtype=float32, numpy=12.947874>,
 <tf.Tensor: shape=(), dtype=float32, numpy=12.327484>]

Q&A#

What happens when I want to fit an already fitted eliobj object?#

eliobj_m2_reload.fit()

# prompt: 
# elicit object is already fitted. 
# Do you want to fit it again and overwrite the results? 
# Press 'n' to stop process and 'y' to continue fitting. 

# user input: n

'Process aborded; eliobj is not re-fitted.'

Can I force re-fitting?#

Sometimes, especially when we only want to test something, it can be inconvenient to repeatedly confirm whether results should be overwritten. To address this issue, you can set overwrite=True to enable re-fitting without any prompts.

eliobj_m2_reload.fit(overwrite=True)

Initialization

100%|██████████| 8/8 [00:01<00:00,  7.43it/s]

 
Training

100%|██████████| 4/4 [00:01<00:00,  2.90it/s]

What happens when I want to save an eliobj object with a name that already exists?#

eliobj_m2_reload.save(name="m2")

# prompt:
# In provided directory exists already a file with identical name. Do you want to overwrite it? 
# Press 'y' for overwriting and 'n' for abording. 

# user input: n

'Process aborded. File is not overwritten.'

Can I force overwriting of file while saving?#

Sometimes, especially when we only want to test something, it can be inconvenient to repeatedly confirm whether the locally stored results file should be overwritten. To address this issue, you can set overwrite=True to enable re-fitting without any prompts.

eliobj_m2_reload.save(name="m2", overwrite=True)

saved in: ./results/parametric_prior/m2_0.pkl