Save and load the elicit object#
Save an unfitted elicit object (e.g., for sharing with collaborators)#
Step 0: Load necessary libraries and functions/classes#
import tensorflow_probability as tfp
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import pandas as pd
import elicit as el
from elicit.extras import utils
tfd = tfp.distributions
# numeric, standardized predictor
def std_predictor(N, quantiles):
X = tf.cast(np.arange(N), tf.float32)
X_std = (X-tf.reduce_mean(X))/tf.math.reduce_std(X)
X_sel = tfp.stats.percentile(X_std, quantiles)
return X_sel
# implemented, generative model
class ToyModel:
def __call__(self, prior_samples, design_matrix, **kwargs):
B = prior_samples.shape[0]
S = prior_samples.shape[1]
# preprocess shape of design matrix
X = tf.broadcast_to(design_matrix[None, None,:],
(B,S,len(design_matrix)))
# linear predictor (= mu)
epred = tf.add(prior_samples[:, :, 0][:,:,None],
tf.multiply(prior_samples[:, :, 1][:,:,None], X)
)
# data-generating model
likelihood = tfd.Normal(
loc=epred, scale=tf.expand_dims(prior_samples[:, :, -1], -1)
)
# prior predictive distribution (=height)
ypred = likelihood.sample()
# selected observations
y_X0, y_X1, y_X2 = (ypred[:,:,0], ypred[:,:,1], ypred[:,:,2])
return dict(
likelihood=likelihood,
ypred=ypred, epred=epred,
prior_samples=prior_samples,
y_X0=y_X0, y_X1=y_X1, y_X2=y_X2
)
Step 1: Create the elicit object#
# define the generative model
model=el.model(
obj=ToyModel,
design_matrix=std_predictor(N=200, quantiles=[25,50,75])
)
# specify the model parameters and their prior distribution families
parameters=[
el.parameter(
name="beta0",
family=tfd.Normal,
hyperparams=dict(
loc=el.hyper("mu0"),
scale=el.hyper("sigma0", lower=0)
)
),
el.parameter(
name="beta1",
family=tfd.Normal,
hyperparams=dict(
loc=el.hyper("mu1"),
scale=el.hyper("sigma1", lower=0) # TODO specify error message
)
),
el.parameter(
name="sigma",
family=tfd.HalfNormal,
hyperparams=dict(
scale=el.hyper("sigma2", lower=0)
)
),
]
# specify the target quantities and corresponding elicitation technique
targets=[
el.target(
name=f"y_X{i}",
query=el.queries.quantiles((5, 25, 50, 75, 95)),
loss=el.losses.MMD2(kernel="energy"),
weight=1.0
) for i in range(3)
]
# use an oracle to simulate a ground truth for the expert data
expert=el.expert.simulator(
ground_truth = {
"beta0": tfd.Normal(loc=5, scale=1),
"beta1": tfd.Normal(loc=2, scale=1),
"sigma": tfd.HalfNormal(scale=10.0),
},
num_samples = 10_000
)
# specify the optimizer for gradient descent
optimizer=el.optimizer(
optimizer=tf.keras.optimizers.Adam,
learning_rate=0.1,
clipnorm=1.0
)
# define the trainer model with model name, used approach, seed, etc.
trainer=el.trainer(
method="parametric_prior",
name="toy0",
seed=0,
epochs=4
)
# specify the initialization distribution, used to draw the initial values
# for the hyperparameters
initializer=el.initializer(
method="sobol",
loss_quantile=0,
iterations=8,
distribution=el.initialization.uniform(
radius=1,
mean=0
)
)
elicit = el.Elicit(
model=model,
parameters=parameters,
targets=targets,
expert=expert,
optimizer=optimizer,
trainer=trainer,
initializer=initializer
)
Step 2: Save the unfitted elicit object#
el.utils.save_elicit(elicit, "./results/elicit_toy0.pkl")
saved elicit as: ./results/elicit_toy0.pkl
Load and fit the unfitted elicit object#
Load the elicit object#
elicit_toy0 = el.utils.load_elicit("./results/elicit_toy0.pkl")
Fit the loaded elicit object#
elicit_toy0.fit(save_dir=None, silent=True)
Initialization
100%|██████████| 8/8 [00:01<00:00, 7.18it/s]
Training
100%|██████████| 4/4 [00:01<00:00, 3.29it/s]
Inspect the fitted elicit object#
results saved for each epoch are stored in
history(type: dictionary)
# information saved in the history object
print(elicit_toy0.history.keys())
dict_keys(['loss', 'loss_component', 'time', 'hyperparameter', 'hyperparameter_gradient'])
results saved only for the last epoch are stored in
results(type: dictionary)
# information saved in the results object
print(elicit_toy0.results.keys())
dict_keys(['target_quantities', 'elicited_statistics', 'prior_samples', 'model_samples', 'model', 'loss_tensor_expert', 'loss_tensor_model', 'expert_elicited_statistics', 'expert_prior_samples', 'init_loss_list', 'init_prior', 'init_matrix'])
Save results on disk#
If you want to save the results locally, you need to specify a directory name through the attribute save_dir. The results are then saved according to the following rule: {save_dir_value}/{method}/{model_name}_{seed}.pkl
Example:
consider the above created
elicitobject (“elicit_toy0”)fit the elicit object with
save_dir="res"results are saved in
./res/parametric_prior/toy0_0.pkl
# Note:
# The silent argument controls whether the history is returned after fitting
# If "silent=True" nothing is returned; if "silent=False" the history is returned.
# by default "silent=False"
hist = elicit_toy0.fit(save_dir="res")
final_res = pd.read_pickle("res/parametric_prior/toy0_0.pkl")
final_res.keys()
Initialization
100%|██████████| 8/8 [00:01<00:00, 6.44it/s]
Training
100%|██████████| 4/4 [00:01<00:00, 3.34it/s]
dict_keys(['history', 'results'])
Save only subset of results#
Sometimes you don’t want to save all possible results but only a relevant subset.
You can control this by the arguments save_configs_history and save_configs_results in the el.trainer callable.
Example
I don’t want to save information about the hyperparameter gradients and the single loss components. This can be done as follows:
elicit = el.Elicit(
model=model,
parameters=parameters,
targets=targets,
expert=expert,
optimizer=optimizer,
trainer=el.trainer(
method="parametric_prior",
name="toy1",
seed=0,
epochs=4,
save_configs_history=el.configs.save_history(hyperparameter_gradient=False, loss_component=False)
),
initializer=initializer
)
# fit elicit obj
elicit.fit(save_dir="res", silent=True)
# inspect saved results
# note that loss_component and hyperparameter_gradient are not saved
pd.read_pickle("res/parametric_prior/toy1_0.pkl")["history"].keys()
Initialization
100%|██████████| 8/8 [00:01<00:00, 7.18it/s]
Training
100%|██████████| 4/4 [00:01<00:00, 2.93it/s]
dict_keys(['loss', 'time', 'hyperparameter'])
Save and reload the fitted elicit object#
Step 1: Save the fitted object#
el.utils.save_elicit(elicit_toy0, "./results/fit_toy0.pkl")
saved elicit as: ./results/fit_toy0.pkl
Step 2: Load and inspect the fitted elicit object#
fit_toy0 = el.utils.load_elicit("./results/fit_toy0.pkl")
fit_toy0.history["loss"]
[<tf.Tensor: shape=(), dtype=float32, numpy=10.571016>,
<tf.Tensor: shape=(), dtype=float32, numpy=10.069366>,
<tf.Tensor: shape=(), dtype=float32, numpy=9.578753>,
<tf.Tensor: shape=(), dtype=float32, numpy=9.098384>]
Q&A#
What happens when I want to fit an already fitted elicit object?#
fit_toy0.fit(save_dir=None)
# prompt:
# elicit object is already fitted.
# Do you want to fit it again and overwrite the results?
# Press 'n' to stop process and 'y' to continue fitting.
# user input: n
'Process aborded; elicit object is not re-fitted.'
Can I force re-fitting?#
Sometimes, especially when we only want to test something, it can be inconvenient to repeatedly confirm whether results should be overwritten. To address this, you can use the force_fit argument in the fit method to enable re-fitting without any prompts.
fit_toy0.fit(save_dir=None, force_fit=True)
What happens when I want to save an elicit object with a name that already exists?#
el.utils.save_elicit(elicit_toy0, "./results/fit_toy0.pkl")
# prompt:
# In provided directory exists already a file with identical name. Do you want to overwrite it?
# Press 'y' for overwriting and 'n' for abording.
# user input: n
'Process aborded. File is not overwritten.'
Can I force overwriting of file while saving?#
Sometimes, especially when we only want to test something, it can be inconvenient to repeatedly confirm whether the locally stored results file should be overwritten. To address this, you can use the force_overwrite argument in the fit method to enable re-fitting without any prompts.
el.utils.save_elicit(elicit_toy0, "./results/fit_toy0.pkl", force_overwrite=True)