{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will study the hyperparamter tuning of fully connected scatternet\n",
    "\n",
    "# first find optimal number of layers and neuron numbers\n",
    "# second optimize the batch size and number of epochs for the best learned architecture\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the dataset here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-04T16:14:58.074796Z",
     "start_time": "2018-09-04T16:14:57.660525Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hegder/anaconda3/envs/deep/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/hegder/anaconda3/envs/deep/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/hegder/anaconda3/envs/deep/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset has been loaded\n",
      "x-train (44999, 8)\n",
      "x-test  (55001, 8)\n",
      "y-train (44999, 256)\n",
      "y-test  (55001, 256)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import h5py\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#now load this dataset \n",
    "h5f = h5py.File('./datasets/s8_sio2tio2_v2.h5','r')\n",
    "X = h5f['sizes'][:]\n",
    "Y = h5f['spectrum'][:]\n",
    "\n",
    "#get the ranges of the loaded data\n",
    "num_layers = X.shape[1]\n",
    "num_lpoints = Y.shape[1]\n",
    "size_max = np.amax(X)\n",
    "size_min = np.amin(X)\n",
    "size_av = 0.5*(size_max + size_min)\n",
    "\n",
    "#this information is not given in the dataset\n",
    "lam_min = 300\n",
    "lam_max = 1200\n",
    "lams = np.linspace(lam_min, lam_max, num_lpoints)\n",
    "\n",
    "#create a train - test split of the dataset\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.55, random_state=42)\n",
    "\n",
    "# normalize inputs \n",
    "x_train = (x_train - 50)/20 \n",
    "x_test = (x_test - 50)/20 \n",
    "\n",
    "print(\"Dataset has been loaded\")\n",
    "print(\"x-train\", x_train.shape)\n",
    "print(\"x-test \", x_test.shape)\n",
    "print(\"y-train\", y_train.shape)\n",
    "print(\"y-test \", y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create models here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T07:21:58.310439Z",
     "start_time": "2018-09-06T07:21:57.952080Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "first (Dense)                (None, 256)               2304      \n",
      "_________________________________________________________________\n",
      "Reshape1 (Reshape)           (None, 4, 64)             0         \n",
      "_________________________________________________________________\n",
      "Up1 (UpSampling1D)           (None, 8, 64)             0         \n",
      "_________________________________________________________________\n",
      "Conv1 (Conv1D)               (None, 8, 64)             28736     \n",
      "_________________________________________________________________\n",
      "Conv2 (Conv1D)               (None, 8, 32)             14368     \n",
      "_________________________________________________________________\n",
      "Conv3 (Conv1D)               (None, 8, 32)             7200      \n",
      "_________________________________________________________________\n",
      "Conv4 (Conv1D)               (None, 8, 32)             7200      \n",
      "_________________________________________________________________\n",
      "Conv5 (Conv1D)               (None, 8, 32)             7200      \n",
      "_________________________________________________________________\n",
      "Conv6 (Conv1D)               (None, 8, 32)             7200      \n",
      "_________________________________________________________________\n",
      "flatten_105 (Flatten)        (None, 256)               0         \n",
      "=================================================================\n",
      "Total params: 74,208\n",
      "Trainable params: 74,208\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import scnets as scn\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "#define and visualize the model here\n",
    "#model = scn.fullycon(num_layers, num_lpoints, 4, 500, 2)\n",
    "\n",
    "model = scn.conv1dmodel(in_size=8, \n",
    "        out_size=256,\n",
    "        c1_nf=64,\n",
    "        clayers=5,\n",
    "        ker_size=7)\n",
    "model.summary()\n",
    "#SVG(model_to_dot(model).create(prog='dot', format='svg'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-04T08:53:28.967090Z",
     "start_time": "2018-09-04T08:42:58.990636Z"
    }
   },
   "outputs": [],
   "source": [
    "x_t, x_v, y_t, y_v = train_test_split(x_train, y_train, test_size=0.2, random_state=42)\n",
    "history = model.fit(x_t, y_t,\n",
    "                    batch_size=32,\n",
    "                    epochs=250, \n",
    "                    verbose=1,\n",
    "                    validation_data=(x_v, y_v))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-04T09:13:20.325358Z",
     "start_time": "2018-09-04T09:13:20.044281Z"
    }
   },
   "outputs": [],
   "source": [
    "scn.plot_training_history(history, 64*2.56)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-09-05T05:27:49.316Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 18 candidates, totalling 54 fits\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.metrics import fbeta_score, make_scorer\n",
    "\n",
    "def my_custom_score_func(ground_truth, predictions):\n",
    "    diff = np.abs(ground_truth - predictions)/np.abs(ground_truth)\n",
    "    return -100*np.mean(diff)\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "import scnets as scn\n",
    "#model = KerasClassifier(build_fn=scn.fullycon, in_size=8, out_size=250, N_gpus=1, epochs=500, verbose=0)\n",
    "\n",
    "model = KerasRegressor(build_fn=scn.conv1dmodel, \n",
    "                        in_size=8, \n",
    "                        out_size=256, \n",
    "                        c1_nf=64,\n",
    "                        clayers=3,\n",
    "                        ker_size=3,\n",
    "                        epochs=250, \n",
    "                        verbose=0)\n",
    "my_score = make_scorer(my_custom_score_func, greater_is_better=False)\n",
    "\n",
    "\n",
    "\n",
    "param_grid = dict(ker_size=[3, 5, 7],\n",
    "                  clayers=[3,4,5],\n",
    "                  batch_size=[32,64])                                  \n",
    "grid = GridSearchCV(estimator=model, \n",
    "                    param_grid=param_grid, \n",
    "                    n_jobs=1, \n",
    "                    scoring=my_score,\n",
    "                    verbose=1)\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T11:13:09.815660Z",
     "start_time": "2018-09-06T11:13:09.803034Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.61959961 0.65246116 0.66099284 0.6769027  0.67712374 0.69844051\n",
      " 0.70291055 0.70903711 0.7103426  0.71306676 0.7421975  0.74365013\n",
      " 0.80096641 0.83683735 0.83717051 0.87958107 0.90866163 0.93305465]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'batch_size': 32, 'clayers': 5, 'ker_size': 7},\n",
       " {'batch_size': 32, 'clayers': 3, 'ker_size': 7},\n",
       " {'batch_size': 32, 'clayers': 4, 'ker_size': 7},\n",
       " {'batch_size': 32, 'clayers': 3, 'ker_size': 5},\n",
       " {'batch_size': 64, 'clayers': 5, 'ker_size': 5},\n",
       " {'batch_size': 64, 'clayers': 3, 'ker_size': 7},\n",
       " {'batch_size': 32, 'clayers': 4, 'ker_size': 5},\n",
       " {'batch_size': 64, 'clayers': 4, 'ker_size': 7},\n",
       " {'batch_size': 64, 'clayers': 5, 'ker_size': 7},\n",
       " {'batch_size': 64, 'clayers': 3, 'ker_size': 5},\n",
       " {'batch_size': 32, 'clayers': 5, 'ker_size': 5},\n",
       " {'batch_size': 64, 'clayers': 4, 'ker_size': 5},\n",
       " {'batch_size': 64, 'clayers': 4, 'ker_size': 3},\n",
       " {'batch_size': 64, 'clayers': 5, 'ker_size': 3},\n",
       " {'batch_size': 32, 'clayers': 3, 'ker_size': 3},\n",
       " {'batch_size': 64, 'clayers': 3, 'ker_size': 3},\n",
       " {'batch_size': 32, 'clayers': 4, 'ker_size': 3},\n",
       " {'batch_size': 32, 'clayers': 5, 'ker_size': 3}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bestidx = np.argsort(grid_result.cv_results_['mean_test_score'])\n",
    "print(grid_result.cv_results_['mean_test_score'][bestidx])\n",
    "parlist = grid_result.cv_results_['params']\n",
    "runtlist = grid_result.cv_results_['mean_fit_time']\n",
    "bestlist = [parlist[indx] for indx in bestidx]\n",
    "runlist = [runtlist[indx]/60.0 for indx in bestidx]\n",
    "bestlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}