import numpy as np
import glob
import os
from opac_mixer.read import ReadOpacChubb
from opac_mixer.emulator import Emulator

import matplotlib.pyplot as plt

from sklearn.metrics import mean_squared_error

from hyperopt import hp, fmin, tpe, Trials, STATUS_OK, STATUS_FAIL

2023-05-02 11:27:33.952020: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.

Tutorial: Hyperparameter tuning

R_10='10'
R_S1='S1'
base = f'{os.environ["pRT_input_data_path"]}/opacities/lines/corr_k'
files_10 = glob.glob(os.path.join(base,f'*_R_{R_10}/*.h5'))
files_S1 = glob.glob(os.path.join(base,f'*_R_{R_S1}/*.h5'))
opac_10 = ReadOpacChubb(files_10)
opac_S1 = ReadOpacChubb(files_S1)

batchsize = int(4e4)

load = False
load_model = False

em = Emulator([opac_10, opac_S1])

em.setup_sampling_grid(approx_batchsize=batchsize)
if not load:
    em.setup_mix(do_parallel=False)
else:
    em.load_data()
em.setup_scaling()

100%|█████████████████████████████████████████████| 6/6 [00:02<00:00,  2.10it/s]
100%|███████████████████████████████████████████| 40/40 [00:03<00:00, 12.61it/s]

deepset_params = {
    'learning_rate': hp.loguniform('learning_rate', np.log(1e-4), np.log(1e-2)),
    'hidden_units': hp.quniform('hidden_units', 8, 32, 2),
}
# Note: We may not be able to use early stopping, since results can be vary bad and that would break the fmin function

para = dict()
para['reg_params'] = deepset_params
para['reg_params']['verbose'] = False
para['loss_func' ] = lambda y, pred: np.sqrt(mean_squared_error(np.log(y), np.log(pred)))

class HPOpt(object):
    """
    Following this example:
    https://towardsdatascience.com/an-example-of-hyperparameter-optimization-on-xgboost-lightgbm-and-catboost-using-hyperopt-12bc41a271e
    """

    def __init__(self, emulator):
        self.emulator = emulator

    def process(self, fn_name, space, trials, algo, max_evals):
        fn = getattr(self, fn_name)
        try:
            result = fmin(fn=fn, space=space, algo=algo, max_evals=max_evals, trials=trials)
        except Exception as e:
            return {'status': STATUS_FAIL,
                    'exception': str(e)}
        return result, trials

    def deepset_reg(self, para):
        print(f'testing {para["reg_params"]}')
        para['reg_params']['hidden_units'] = int(para['reg_params']['hidden_units'])
        self.emulator.setup_model(**para['reg_params'])
        self.emulator.fit()
        pred = self.emulator.predict(self.emulator.X_test)
        loss = para['loss_func'](self.emulator.y_test, pred)
        print(loss)
        return {'loss': loss, 'status': STATUS_OK}

trials = Trials()

obj = HPOpt(em)

opt = obj.process(fn_name='deepset_reg', space=para, trials=trials, algo=tpe.suggest, max_evals=100)

testing {'hidden_units': 22.0, 'learning_rate': 0.009639427937547154, 'verbose': False}
0.8498246682000619
testing {'hidden_units': 28.0, 'learning_rate': 0.0007222199018827587, 'verbose': False}
0.6584006373828777
testing {'hidden_units': 16.0, 'learning_rate': 0.00042633055604179577, 'verbose': False}
0.5539265943469254
testing {'hidden_units': 32.0, 'learning_rate': 0.001360822606344268, 'verbose': False}
0.5127579313690871
testing {'hidden_units': 14.0, 'learning_rate': 0.0009791716540062023, 'verbose': False}
0.5681199037225347
testing {'hidden_units': 30.0, 'learning_rate': 0.004849545424759847, 'verbose': False}
0.6095323677716094
testing {'hidden_units': 26.0, 'learning_rate': 0.0017357309338336502, 'verbose': False}
0.5427402545973544
testing {'hidden_units': 30.0, 'learning_rate': 0.002304127318727969, 'verbose': False}
0.5752927451569829
testing {'hidden_units': 24.0, 'learning_rate': 0.002973832496326314, 'verbose': False}
0.5561454030540911
testing {'hidden_units': 22.0, 'learning_rate': 0.001086873795666205, 'verbose': False}
0.5671948296100852
testing {'hidden_units': 32.0, 'learning_rate': 0.0007383167277272579, 'verbose': False}
0.4895793657410836
testing {'hidden_units': 30.0, 'learning_rate': 0.0007711311816257909, 'verbose': False}
0.4996077960101605
testing {'hidden_units': 32.0, 'learning_rate': 0.00035131776545554874, 'verbose': False}
0.684279160389004
testing {'hidden_units': 30.0, 'learning_rate': 0.0005835802572683012, 'verbose': False}
0.7244328603750352
testing {'hidden_units': 32.0, 'learning_rate': 0.0004681703213696952, 'verbose': False}
0.5089783795136884
testing {'hidden_units': 32.0, 'learning_rate': 0.0008228590012195502, 'verbose': False}
0.5090533190976656
testing {'hidden_units': 30.0, 'learning_rate': 0.0013164610535313453, 'verbose': False}
0.5157210356443074
testing {'hidden_units': 26.0, 'learning_rate': 0.0006797540894490644, 'verbose': False}
0.49799807864303647
testing {'hidden_units': 26.0, 'learning_rate': 0.0003057840986349148, 'verbose': False}
0.7374238828971357
testing {'hidden_units': 28.0, 'learning_rate': 0.0004009829664007632, 'verbose': False}
0.8363024023852144
testing {'hidden_units': 24.0, 'learning_rate': 0.0006109623683409882, 'verbose': False}
0.6218175782270138
testing {'hidden_units': 28.0, 'learning_rate': 0.00023291177957504753, 'verbose': False}
0.7885310899400488
testing {'hidden_units': 26.0, 'learning_rate': 0.0001834934587102731, 'verbose': False}
0.6481837307901244
testing {'hidden_units': 32.0, 'learning_rate': 0.0004770766928563477, 'verbose': False}
0.6691736519706238
testing {'hidden_units': 18.0, 'learning_rate': 0.0007043571757111523, 'verbose': False}
0.5562201728279398
testing {'hidden_units': 26.0, 'learning_rate': 0.0011392629322666086, 'verbose': False}
0.5370231228384003
testing {'hidden_units': 30.0, 'learning_rate': 0.001739832525547357, 'verbose': False}
0.5409175549732536
testing {'hidden_units': 28.0, 'learning_rate': 0.0010303526013455402, 'verbose': False}
0.5146158575832172
testing {'hidden_units': 22.0, 'learning_rate': 0.0003289051337715824, 'verbose': False}
0.7116270193475916
testing {'hidden_units': 32.0, 'learning_rate': 0.00240782918951775, 'verbose': False}
0.5505588153567728
testing {'hidden_units': 24.0, 'learning_rate': 0.0005214417196159253, 'verbose': False}
0.63015556828553
testing {'hidden_units': 28.0, 'learning_rate': 0.00011912826586868648, 'verbose': False}
0.7763869930841739
testing {'hidden_units': 30.0, 'learning_rate': 0.00026558804561660336, 'verbose': False}
0.6974434485341271
testing {'hidden_units': 16.0, 'learning_rate': 0.0009582337941846697, 'verbose': False}
0.5543927209227959
testing {'hidden_units': 20.0, 'learning_rate': 0.0012928079889993735, 'verbose': False}
0.5262687372275326
testing {'hidden_units': 26.0, 'learning_rate': 0.0006631658178864239, 'verbose': False}
0.7779101127891651
testing {'hidden_units': 22.0, 'learning_rate': 0.00016339556249999083, 'verbose': False}
0.7310155557687833
testing {'hidden_units': 30.0, 'learning_rate': 0.0033430226753002417, 'verbose': False}
0.5644128539971887
testing {'hidden_units': 18.0, 'learning_rate': 0.00037390147899707567, 'verbose': False}
0.6846452213074297
testing {'hidden_units': 32.0, 'learning_rate': 0.0004455392794575827, 'verbose': False}
0.6902648045482419
testing {'hidden_units': 12.0, 'learning_rate': 0.0015276331071246386, 'verbose': False}
0.5709071856563904
testing {'hidden_units': 28.0, 'learning_rate': 0.0017206640198669648, 'verbose': False}
0.5466856314244408
testing {'hidden_units': 26.0, 'learning_rate': 0.00020724649009308419, 'verbose': False}
0.6434068670888469
testing {'hidden_units': 32.0, 'learning_rate': 0.0005745495342869347, 'verbose': False}
0.724378918692089
testing {'hidden_units': 20.0, 'learning_rate': 0.0009027920005944342, 'verbose': False}
0.535292005745887
 99%|███████▉| 99/100 [55:28<01:13, 73.97s/trial, best loss: 0.4895793657410836]

trials.best_trial

{'state': 2,
 'tid': 65,
 'spec': None,
 'result': {'loss': 0.4895793657410836, 'status': 'ok'},
 'misc': {'tid': 65,
  'cmd': ('domain_attachment', 'FMinIter_Domain'),
  'workdir': None,
  'idxs': {'hidden_units': [65], 'learning_rate': [65]},
  'vals': {'hidden_units': [32.0], 'learning_rate': [0.0007383167277272579]}},
 'exp_key': None,
 'owner': None,
 'version': 0,
 'book_time': datetime.datetime(2023, 5, 2, 10, 52, 9, 245000),
 'refresh_time': datetime.datetime(2023, 5, 2, 10, 53, 59, 519000)}

losses = np.asarray(trials.losses(), dtype=np.float_)
hist_cts, hist_edges = np.histogram(losses[np.isfinite(losses)])
plt.hist(losses[np.isfinite(losses)], bins=20)

(array([10., 13., 18.,  8.,  2.,  7.,  4.,  6.,  8.,  8.,  3.,  6.,  0.,
         2.,  1.,  1.,  0.,  0.,  0.,  2.]),
 array([0.48957937, 0.51531799, 0.54105661, 0.56679524, 0.59253386,
        0.61827249, 0.64401111, 0.66974974, 0.69548836, 0.72122699,
        0.74696561, 0.77270423, 0.79844286, 0.82418148, 0.84992011,
        0.87565873, 0.90139736, 0.92713598, 0.9528746 , 0.97861323,
        1.00435185]),
 <BarContainer object of 20 artists>)

plt.plot(trials.losses())

[<matplotlib.lines.Line2D at 0x168d88b80>]

plt.scatter(trials.vals['learning_rate'], trials.losses(), )
plt.xscale('log')
plt.show()

plt.scatter(trials.vals['hidden_units'], trials.losses(), )
plt.show()

import pickle

with open('data/hyperopt.pickle', 'wb') as f:
    pickle.dump(trials, f)