{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "### Dataset Loading and testing" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T16:23:20.551161Z", "start_time": "2018-09-30T16:23:19.956959Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Dataset has been loaded\n", "x-train (60000, 8)\n", "x-test (40000, 8)\n", "y-train (60000, 256)\n", "y-test (40000, 256)\n" ] } ], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import h5py\n", "from sklearn.model_selection import train_test_split\n", "#import jtplot submodule from jupyterthemes\n", "from jupyterthemes import jtplot\n", "#currently installed theme will be used to\n", "#set plot style if no arguments provided\n", "jtplot.style()\n", "\n", "#now load this dataset \n", "h5f = h5py.File('./datasets/s8_sio2tio2_v2.h5','r')\n", "X = h5f['sizes'][:]\n", "Y = h5f['spectrum'][:]\n", "\n", "\n", "\n", "\n", "#get the ranges of the loaded data\n", "num_layers = X.shape[1]\n", "num_lpoints = Y.shape[1]\n", "size_max = np.amax(X)\n", "size_min = np.amin(X)\n", "size_av = 0.5*(size_max + size_min)\n", "\n", "#this information is not given in the dataset\n", "lam_min = 300\n", "lam_max = 1200\n", "lams = np.linspace(lam_min, lam_max, num_lpoints)\n", "\n", "# X = np.expand_dims(X, 1)\n", "# #X = np.expand_dims(X, 3)\n", "# Y = np.expand_dims(Y, 1)\n", "# #Y = np.expand_dims(Y, 3)\n", "\n", "\n", "\n", "\n", "\n", "\n", "#create a train - test split of the dataset\n", "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.4, random_state=42)\n", "\n", "# normalize inputs \n", "x_train = (x_train - 50)/20 \n", "x_test = (x_test - 50)/20 \n", "\n", "print(\"Dataset has been loaded\")\n", "print(\"x-train\", x_train.shape)\n", "print(\"x-test \", x_test.shape)\n", "print(\"y-train\", y_train.shape)\n", "print(\"y-test \", y_test.shape)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Model Development" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-27T05:04:59.838628Z", "start_time": "2018-09-27T05:04:59.808085Z" } }, "outputs": [], "source": [ "from keras.utils import to_channels_first\n", "x_train = to_channels_first(x_train)\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T16:23:29.926776Z", "start_time": "2018-09-30T16:23:29.143314Z" }, "code_folding": [ 48, 65, 84, 123, 169, 234 ] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using MXNet backend\n" ] } ], "source": [ "from keras import backend as K\n", "from keras.models import Sequential, Model\n", "from keras.layers import Dense, Dropout, Reshape, UpSampling1D, Conv1D, Flatten, Activation\n", "from keras.utils import np_utils, multi_gpu_model\n", "from keras.regularizers import l2\n", "from keras.wrappers.scikit_learn import KerasRegressor\n", "from keras.optimizers import Adam\n", "from keras.layers.normalization import BatchNormalization\n", "from keras.layers import PReLU\n", "\n", "\n", "from sklearn.model_selection import cross_val_score, KFold\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.pipeline import Pipeline\n", "\n", "num_gpus = 2\n", "gpu_list = [\"gpu(%d)\" % i for i in range(num_gpus)]\n", "\n", "\n", "\n", "#define various models here\n", "#naive percentage loss\n", "def size_percent_loss(y_true, y_pred):\n", " y_true_a = 0.5*y_true*(size_max - size_min) + size_av\n", " y_pred_a = 0.5*y_pred*(size_max - size_min) + size_av\n", " y_err = np.abs(y_true_a - y_pred_a)/y_true_a\n", " y_err_f = K.flatten(y_err)\n", " return K.sum(y_err_f)\n", "\n", "#naive percentage loss\n", "def naive_percent_loss(y_true, y_pred):\n", " y_err = np.abs(y_true - y_pred)/y_true\n", " y_err_f = K.flatten(y_err)\n", " return K.sum(y_err_f)\n", "\n", "\n", "#function to test performance on testset \n", "def calc_mre(y_true, y_pred):\n", " y_err = 100*np.abs(y_true - y_pred)/y_true\n", " return np.mean(y_err)\n", "\n", "#function to test performance on testset \n", "def calc_mre_K(y_true, y_pred):\n", " y_err = 100*np.abs(y_true - y_pred)/y_true\n", " return K.mean(y_err)\n", "\n", "\n", "\n", "def naiveploss_mgpu_model():\n", " # create model\n", " model = Sequential()\n", " model = multi_gpu_model(model, gpus=num_gpus)\n", " model.add(Dense(250, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='first' ))\n", " model.add(Dense(250, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='second' ))\n", " model.add(Dense(250, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='third' ))\n", " model.add(Dense(250, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='fourth' ))\n", " model.add(Dense(250, kernel_initializer='normal', name='last'))\n", " # Compile model\n", " model.compile(loss=naive_percent_loss, optimizer='adam', context = gpu_list)\n", " return model\n", "\n", "def naiveploss_model():\n", " # create model\n", " model = Sequential()\n", " model.add(Dense(256, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='first' ))\n", " model.add(Dense(256, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='second' ))\n", " model.add(Dense(256, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='third' ))\n", " model.add(Dense(256, input_dim=x_train.shape[1], kernel_initializer='normal', activation='relu', \n", " name='fourth' ))\n", " model.add(Dense(256, kernel_initializer='normal', name='last'))\n", " # Compile model\n", " model.compile(loss=naive_percent_loss, optimizer='adam', metrics=['accuracy'])\n", " return model\n", "\n", "import timeit\n", "#here we must have a function that calls the training routine n times and then gives avg and stddev \n", "# of the resulting figures\n", "def net_performance(modelfunc, num_trials=3, batch_size=32, num_epochs=200, num_gpus=2):\n", " models = []\n", " train_err = np.ones(num_trials)\n", " test_err = np.ones(num_trials)\n", " val_err = np.ones(num_trials)\n", " train_time = np.ones(num_trials)\n", " for tnum in np.arange(num_trials):\n", " print(\"iteration: \" + str(tnum + 1))\n", " model_curr = modelfunc()\n", " x_t, x_v, y_t, y_v = train_test_split(x_train, y_train, test_size=0.2, random_state=42)\n", " start_time = timeit.default_timer()\n", " history = model_curr.fit(x_t, y_t,\n", " batch_size=batch_size*num_gpus,\n", " epochs=num_epochs, \n", " verbose=1,\n", " validation_data=(x_v, y_v))\n", " train_time[tnum] = timeit.default_timer() - start_time\n", " models.append(model_curr)\n", " train_err[tnum] = (100.0/num_lpoints)*history.history['loss'][-1]/(batch_size*num_gpus)\n", " val_err[tnum] = (100.0/num_lpoints)*history.history['val_loss'][-1]/(batch_size*num_gpus)\n", " test_err[tnum] = calc_mre(y_test, models[tnum].predict(x_test))\n", " return train_err, val_err, test_err, train_time\n", "\n", "#staging area for new models \n", "def plot_training_history(history, factor):\n", " loss, val_loss = history.history['loss'], history.history['val_loss']\n", " loss = np.asarray(loss)/(factor)\n", " val_loss = np.asarray(val_loss)/(factor)\n", " epochs = len(loss)\n", " \n", " fig, axs = plt.subplots(1,1, figsize=(5,2.5))\n", " axs.semilogy(np.arange(1, epochs + 1), loss, label='Train error')\n", " axs.semilogy(np.arange(1, epochs + 1), val_loss, label='Test error')\n", " axs.set_xlabel('Epoch number')\n", " #axs.set_ylim((0.4, 3))\n", " axs.set_xlim(left=1)\n", "# plt.yticks(np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.5, 2]), \n", "# ('0.1', '0.2', '0.3', '0.4', '0.5', '0.75', '1.0', '1.5', '2'))\n", " plt.yticks(np.array([0.5, 0.75, 1.0, 1.5, 2]), \n", " ('0.5', '0.75', '1.0', '1.5', '2'))\n", " axs.set_ylabel('MRE (%)')\n", " axs.legend(loc=\"best\")\n", " fig.savefig(\"foo2.pdf\", bbox_inches='tight')\n", "\n", "from keras.utils import to_channels_first\n", "\n", "def conv1d_lkyrelu():\n", " \n", " #gpu_list = [\"gpu(%d)\" % i for i in range(num_gpus)]\n", " \n", " # create model\n", " model = Sequential()\n", " \n", " model.add(Dense(256, input_dim=8, kernel_initializer='normal', \n", " name='first'))\n", " #model.add(BatchNormalization())\n", " #model.add(Activation('relu')) \n", " model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None))\n", " \n", " model.add(Reshape((4, 64)))\n", " model.add(UpSampling1D(size=2))\n", " \n", " \n", " model.add(Conv1D(filters=64, kernel_size=3, strides=1, padding='same', \n", " dilation_rate=1, \n", " kernel_initializer='normal'))\n", " model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None))\n", " \n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None))\n", " \n", " model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", " dilation_rate=1, \n", " kernel_initializer='normal'))\n", " model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None))\n", "\n", "\n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(PReLU(alpha_initializer='ones', alpha_regularizer=None)) \n", " \n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(PReLU(alpha_initializer='ones', alpha_regularizer=None)) \n", " \n", " model.add(Flatten())\n", " # Compile model\n", " model.compile(loss=naive_percent_loss, optimizer='adam', metrics=[calc_mre_K])\n", " return model\n", " \n", " \n", "def conv1d_model_bnorm():\n", " \n", " #gpu_list = [\"gpu(%d)\" % i for i in range(num_gpus)]\n", " \n", " # create model\n", " model = Sequential()\n", " \n", " model.add(Dense(256, input_dim=8, kernel_initializer='normal', \n", " name='first' ))\n", " #model.add(BatchNormalization())\n", " model.add(Activation('relu'))\n", " #model.add(Dropout(0.2))\n", " \n", " model.add(Reshape((4, 64)))\n", " model.add(UpSampling1D(size=2))\n", " \n", " model.add(Conv1D(filters=64, kernel_size=3, strides=1, padding='same', \n", " dilation_rate=1, kernel_initializer='normal'))\n", " #model.add(BatchNormalization())\n", " model.add(Activation('relu'))\n", " #model.add(Dropout(0.2))\n", " #model.add(UpSampling1D(size=5))\n", "\n", " model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", " dilation_rate=1, kernel_initializer='normal'))\n", " model.add(Activation('relu'))\n", " #model.add(Dropout(0.3))\n", " \n", "# model.add(Conv1D(filters=64, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(Activation('relu')) \n", "# model.add(Dropout(0.3))\n", "\n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(Activation('relu')) \n", " \n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(Activation('relu')) \n", " \n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(Activation('relu')) \n", " \n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, kernel_initializer='normal'))\n", "# model.add(Activation('relu')) \n", " \n", " model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", " dilation_rate=1, kernel_initializer='normal'))\n", " model.add(Activation('relu')) \n", " \n", " \n", " \n", " model.add(Flatten())\n", " # Compile model\n", "# if num_gpus == 1:\n", " model.compile(loss=naive_percent_loss, optimizer='adam', metrics=[calc_mre_K])\n", "# else:\n", "# model.compile(loss=naive_percent_loss, optimizer='adam', metrics=['accuracy'], context = gpu_list)\n", " \n", " \n", " return model \n", "\n", "def resnetb():\n", " model = Sequential()\n", " \n", " #first layer6\n", " model.add(Dense(256, input_dim=8, kernel_initializer='normal', \n", " name='first'))\n", " model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None))\n", " model.add(Reshape((8, 32)))\n", " \n", " \n", " #resnet block\n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, \n", "# kernel_initializer='normal'))\n", "# model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None))\n", "# model.add(Conv1D(filters=32, kernel_size=3, strides=1, padding='same', \n", "# dilation_rate=1, \n", "# kernel_initializer='normal'))\n", " \n", " \n", " \n", " #Last layer\n", " model.add(Flatten())\n", " \n", " #compile model\n", " model.compile(loss=naive_percent_loss, optimizer='adam', metrics=[calc_mre_K])\n", " \n", " return model\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Model testing" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T16:23:49.159863Z", "start_time": "2018-09-30T16:23:42.827817Z" }, "code_folding": [], "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/hegder/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/backend/mxnet_backend.py:89: UserWarning: MXNet Backend performs best with `channels_first` format. Using `channels_last` will significantly reduce performance due to the Transpose operations. For performance improvement, please use this API`keras.utils.to_channels_first(x_input)`to transform `channels_last` data to `channels_first` format and also please change the `image_data_format` in `keras.json` to `channels_first`.Note: `x_input` is a Numpy tensor or a list of Numpy tensorRefer to: https://github.com/awslabs/keras-apache-mxnet/tree/master/docs/mxnet_backend/performance_guide.md\n", " train_symbol = func(*args, **kwargs)\n", "/home/hegder/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/backend/mxnet_backend.py:92: UserWarning: MXNet Backend performs best with `channels_first` format. Using `channels_last` will significantly reduce performance due to the Transpose operations. For performance improvement, please use this API`keras.utils.to_channels_first(x_input)`to transform `channels_last` data to `channels_first` format and also please change the `image_data_format` in `keras.json` to `channels_first`.Note: `x_input` is a Numpy tensor or a list of Numpy tensorRefer to: https://github.com/awslabs/keras-apache-mxnet/tree/master/docs/mxnet_backend/performance_guide.md\n", " test_symbol = func(*args, **kwargs)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", "input_1 (InputLayer) (None, 8) 0 \n", "__________________________________________________________________________________________________\n", "dense_1 (Dense) (None, 256) 2304 input_1[0][0] \n", "__________________________________________________________________________________________________\n", "p_re_lu_1 (PReLU) (None, 256) 256 dense_1[0][0] \n", "__________________________________________________________________________________________________\n", "reshape_1 (Reshape) (None, 8, 32) 0 p_re_lu_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_1 (Conv1D) (None, 8, 32) 3104 reshape_1[0][0] \n", "__________________________________________________________________________________________________\n", "activation_1 (Activation) (None, 8, 32) 0 conv1d_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_2 (Conv1D) (None, 8, 32) 3104 activation_1[0][0] \n", "__________________________________________________________________________________________________\n", "add_1 (Add) (None, 8, 32) 0 conv1d_2[0][0] \n", " reshape_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_3 (Conv1D) (None, 8, 32) 3104 add_1[0][0] \n", "__________________________________________________________________________________________________\n", "activation_2 (Activation) (None, 8, 32) 0 conv1d_3[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_4 (Conv1D) (None, 8, 32) 3104 activation_2[0][0] \n", "__________________________________________________________________________________________________\n", "add_2 (Add) (None, 8, 32) 0 conv1d_4[0][0] \n", " add_1[0][0] \n", "__________________________________________________________________________________________________\n", "flatten_1 (Flatten) (None, 256) 0 add_2[0][0] \n", "==================================================================================================\n", "Total params: 14,976\n", "Trainable params: 14,976\n", "Non-trainable params: 0\n", "__________________________________________________________________________________________________\n", "Train on 48000 samples, validate on 12000 samples\n", "Epoch 1/2000\n", " 1984/48000 [>.............................] - ETA: 33s - loss: 4030.1485 - calc_mre_K: 49.1961" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/hegder/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/module/bucketing_module.py:408: UserWarning: Optimizer created manually outside Module but rescale_grad is not normalized to 1.0/batch_size/num_workers (1.0 vs. 0.03125). Is this intended?\n", " force_init=force_init)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "48000/48000 [==============================] - 5s 105us/step - loss: 725.7694 - calc_mre_K: 8.8595 - val_loss: 311.2114 - val_calc_mre_K: 3.7990\n", "Epoch 2/2000\n", "11936/48000 [======>.......................] - ETA: 2s - loss: 300.9149 - calc_mre_K: 3.6733" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m validation_data=(x_v, y_v))\n\u001b[0m\u001b[1;32m 84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m 1703\u001b[0m \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1704\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m 1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1707\u001b[0m def evaluate(self, x=None, y=None,\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m 1234\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1238\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/backend/mxnet_backend.py\u001b[0m in \u001b[0;36mtrain_function\u001b[0;34m(inputs)\u001b[0m\n\u001b[1;32m 4566\u001b[0m provide_data=data_shapes, provide_label=label_shapes)\n\u001b[1;32m 4567\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_backward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4568\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4569\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_updates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4570\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_weights_dirty\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/module/bucketing_module.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 474\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinded\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams_initialized\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_initialized\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_params_dirty\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_curr_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 478\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_outputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmerge_multi_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/module/module.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 669\u001b[0m \u001b[0mnum_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 670\u001b[0m \u001b[0mkvstore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_kvstore\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 671\u001b[0;31m param_names=self._exec_group.param_names)\n\u001b[0m\u001b[1;32m 672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 673\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_outputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmerge_multi_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/model.py\u001b[0m in \u001b[0;36m_update_params\u001b[0;34m(param_arrays, grad_arrays, updater, num_device, kvstore, param_names)\u001b[0m\n\u001b[1;32m 174\u001b[0m \u001b[0;31m# use a better solution later\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 176\u001b[0;31m \u001b[0mupdater\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnum_device\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/optimizer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, index, grad, weight)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_state_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstates\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstates_synced\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1517\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_multi_precision\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstates\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1519\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msync_state_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/optimizer.py\u001b[0m in \u001b[0;36mupdate_multi_precision\u001b[0;34m(self, index, weight, grad, state)\u001b[0m\n\u001b[1;32m 272\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight_master_copy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 273\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 274\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_learning_rate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/optimizer.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, index, weight, grad, state)\u001b[0m\n\u001b[1;32m 1083\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1084\u001b[0m adam_update(weight, grad, mean, var, out=weight,\n\u001b[0;32m-> 1085\u001b[0;31m lazy_update=self.lazy_update, lr=lr, wd=wd, **kwargs)\n\u001b[0m\u001b[1;32m 1086\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1087\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/ndarray/register.py\u001b[0m in \u001b[0;36madam_update\u001b[0;34m(weight, grad, mean, var, lr, beta1, beta2, epsilon, wd, rescale_grad, clip_gradient, lazy_update, out, name, **kwargs)\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/_ctypes/ndarray.py\u001b[0m in \u001b[0;36m_imperative_invoke\u001b[0;34m(handle, ndargs, keys, vals, out)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0mc_str_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0mc_str_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvals\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m ctypes.byref(out_stypes)))\n\u001b[0m\u001b[1;32m 93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moriginal_output\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "%autoreload\n", "# import warnings\n", "# warnings.filterwarnings('ignore')\n", "\n", "\n", "# model = resnetb()\n", "# #model = conv1d_lkyrelu()\n", "# #model = conv1d_model_bnorm()\n", "# #model = conv1d_model(1)\n", "\n", "# #model = naiveploss_model()\n", "# model.summary()\n", "\n", " \n", "# from IPython.display import SVG\n", "# from keras.utils.vis_utils import model_to_dot\n", "\n", "# #SVG(model_to_dot(model).create(prog='dot', format='svg'))\n", " \n", "\n", "\n", " \n", " \n", "import scnets as scn\n", "\n", "model = scn.resnet(in_size=8, \n", " out_size=256,\n", " num_units=2,\n", " red_dim=16,\n", " batch_size=32,\n", " ker_size=3)\n", "\n", "# model = scn.conv1dmodel(in_size=8, \n", "# out_size=256,\n", "# batch_size=64,\n", "# c1_nf=64,\n", "# clayers=4,\n", "# ker_size=5)\n", "\n", "# model = scn.convprel(in_size=8, \n", "# out_size=256,\n", "# batch_size=64,\n", "# c1_nf=64,\n", "# clayers=4,\n", "# ker_size=5)\n", "\n", "# model = scn.fullycon(in_size=8, \n", "# out_size=256, \n", "# batch_size=64,\n", "# N_hidden=3, \n", "# N_neurons=36, \n", "# N_gpus=1)\n", "\n", "\n", "\n", "\n", "\n", "# from keras import optimizers\n", "# sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n", "#model.compile(loss=naive_percent_loss, optimizer='nadam', metrics=[calc_mre_K])\n", "\n", "from IPython.display import SVG\n", "from keras.utils.vis_utils import model_to_dot\n", "from keras.utils.vis_utils import plot_model\n", "\n", "SVG(model_to_dot(model, show_shapes=False, show_layer_names=False).create(prog='dot', format='svg'))\n", "\n", "\n", "plot_model(model, show_shapes=False, show_layer_names=False, to_file='model_resnet.svg')\n", "\n", "\n", "\n", "model.summary() \n", " \n", " \n", "x_t, x_v, y_t, y_v = train_test_split(x_train, y_train, test_size=0.2, random_state=42)\n", "# model = naiveploss_mgpu_model()\n", "# model.summary() \n", "history = model.fit(x_t, y_t,\n", " batch_size=32,\n", " epochs=2000, \n", " verbose=1,\n", " validation_data=(x_v, y_v))\n", "\n", "\n", "\n", "\n", "\n", "# y_pred = model.predict(x_test)\n", "# print(calc_mre(y_test, y_pred))\n", "\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T16:23:59.012349Z", "start_time": "2018-09-30T16:23:58.711234Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/hegder/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/backend/mxnet_backend.py:89: UserWarning: MXNet Backend performs best with `channels_first` format. Using `channels_last` will significantly reduce performance due to the Transpose operations. For performance improvement, please use this API`keras.utils.to_channels_first(x_input)`to transform `channels_last` data to `channels_first` format and also please change the `image_data_format` in `keras.json` to `channels_first`.Note: `x_input` is a Numpy tensor or a list of Numpy tensorRefer to: https://github.com/awslabs/keras-apache-mxnet/tree/master/docs/mxnet_backend/performance_guide.md\n", " train_symbol = func(*args, **kwargs)\n", "/home/hegder/anaconda3/envs/dp2/lib/python3.5/site-packages/keras/backend/mxnet_backend.py:92: UserWarning: MXNet Backend performs best with `channels_first` format. Using `channels_last` will significantly reduce performance due to the Transpose operations. For performance improvement, please use this API`keras.utils.to_channels_first(x_input)`to transform `channels_last` data to `channels_first` format and also please change the `image_data_format` in `keras.json` to `channels_first`.Note: `x_input` is a Numpy tensor or a list of Numpy tensorRefer to: https://github.com/awslabs/keras-apache-mxnet/tree/master/docs/mxnet_backend/performance_guide.md\n", " test_symbol = func(*args, **kwargs)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", "input_1 (InputLayer) (None, 8) 0 \n", "__________________________________________________________________________________________________\n", "dense_1 (Dense) (None, 256) 2304 input_1[0][0] \n", "__________________________________________________________________________________________________\n", "p_re_lu_1 (PReLU) (None, 256) 256 dense_1[0][0] \n", "__________________________________________________________________________________________________\n", "reshape_1 (Reshape) (None, 8, 32) 0 p_re_lu_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_1 (Conv1D) (None, 8, 32) 3104 reshape_1[0][0] \n", "__________________________________________________________________________________________________\n", "activation_1 (Activation) (None, 8, 32) 0 conv1d_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_2 (Conv1D) (None, 8, 32) 3104 activation_1[0][0] \n", "__________________________________________________________________________________________________\n", "add_1 (Add) (None, 8, 32) 0 conv1d_2[0][0] \n", " reshape_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_3 (Conv1D) (None, 8, 32) 3104 add_1[0][0] \n", "__________________________________________________________________________________________________\n", "activation_2 (Activation) (None, 8, 32) 0 conv1d_3[0][0] \n", "__________________________________________________________________________________________________\n", "conv1d_4 (Conv1D) (None, 8, 32) 3104 activation_2[0][0] \n", "__________________________________________________________________________________________________\n", "add_2 (Add) (None, 8, 32) 0 conv1d_4[0][0] \n", " add_1[0][0] \n", "__________________________________________________________________________________________________\n", "flatten_1 (Flatten) (None, 256) 0 add_2[0][0] \n", "==================================================================================================\n", "Total params: 14,976\n", "Trainable params: 14,976\n", "Non-trainable params: 0\n", "__________________________________________________________________________________________________\n" ] } ], "source": [ "import scnets as scn\n", "from keras.models import load_model\n", "from scnets import relerr_loss, calc_mre_K\n", "\n", "#Creates a HDF5 file 'my_model.h5'\n", "#odel.save('res15k_model.h5')\n", "#odel = load_model('model/multi_task/try.h5', custom_objects={'loss_max': loss_max})\n", "model = load_model('res15k_model.h5', custom_objects={'relerr_loss': relerr_loss, \n", " 'calc_mre_K' :calc_mre_K})\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T10:30:16.919436Z", "start_time": "2018-09-30T10:30:16.884232Z" }, "scrolled": true }, "outputs": [], "source": [ "# Import the save_mxnet_model API\n", "from keras.models import save_mxnet_model\n", "save_mxnet_model(model=model, prefix='my_mod', epoch=0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T10:30:53.029361Z", "start_time": "2018-09-30T10:30:52.979098Z" } }, "outputs": [], "source": [ "import numpy as np\n", "import mxnet as mx\n", "\n", "# Step1: Load the model in MXNet\n", "\n", "# Use the same prefix and epoch parameters we used in save_mxnet_model API.\n", "sym, arg_params, aux_params = mx.model.load_checkpoint(prefix='my_mod', epoch=0)\n", "\n", "# We use the data_names and data_shapes returned by save_mxnet_model API.\n", "mod = mx.mod.Module(symbol=sym, \n", " data_names=['/input_12'], \n", " context=mx.gpu(), \n", " label_names=None)\n", "mod.bind(for_training=False, \n", " data_shapes=[('/input_12', (1,8))], \n", " label_shapes=mod._label_shapes)\n", "mod.set_params(arg_params, aux_params, allow_missing=True)\n", "\n", "data_iter = mx.io.NDArrayIter(x_test, None, 1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T10:40:42.350586Z", "start_time": "2018-09-30T10:40:42.321089Z" } }, "outputs": [], "source": [ "size = np.random.randint(30, 71, (1,8))\n", "#spec_ac = snlay.calc_spectrum(size, mats, lams)\n", "size = (size - 50.0)/20.0\n", "#size = np.expand_dims(size, axis = 0)\n", "size.shape\n", "res1 = model.predict(size)\n", "data_iter = mx.io.NDArrayIter(size, None, 1)\n", "res2 = mod.predict(data_iter)\n", "\n", "# #y_pred = result.asnumpy()\n", "# print(calc_mre(y_test, y_pred))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T11:18:00.192453Z", "start_time": "2018-09-30T11:18:00.168150Z" } }, "outputs": [], "source": [ "import mxnet as mx\n", "from mxnet import nd\n", "mx.random.seed(1)\n", "\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T16:24:20.697887Z", "start_time": "2018-09-30T16:24:18.314292Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/hegder/anaconda3/envs/dp2/lib/python3.5/site-packages/mxnet/module/bucketing_module.py:408: UserWarning: Optimizer created manually outside Module but rescale_grad is not normalized to 1.0/batch_size/num_workers (1.0 vs. 0.03125). Is this intended?\n", " force_init=force_init)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "0.8106580427940897\n" ] } ], "source": [ "# idx = 102\n", "# print(\"Predicted - \", np.argmax(result[idx].asnumpy()))\n", "# print(\"Actual - \", y_test[idx])\n", "\n", "#result.asnumpy()[1] - y_test[1]\n", "\n", "\n", "y_pred = model.predict(x_test)\n", "print(calc_mre(y_test, y_pred))\n", "#print(calc_mre(y_test, result.asnumpy()))\n", "#history60 = history\n", "#plot_training_history(history, 32*2.56)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "start_time": "2018-09-30T16:15:38.234Z" } }, "outputs": [], "source": [ "from matplotlib import gridspec\n", "import snlay as snlay\n", "#here we test edge cases where failure happens\n", "#size = np.array([70, 60, 50, 40, 30, 20, 10, 10])\n", "#size = np.array([70, 70, 70, 70, 70, 70, 70, 70])\n", "#size = np.array([30, 30, 30, 30, 30, 30, 30, 30])\n", "\n", "#size = np.array([65, 65, 65, 65, 55, 65, 35, 65])\n", "#size = np.array([65, 35, 45, 35, 45, 35, 45, 35])\n", "\n", "size = np.random.randint(30, 71, 8)\n", "mats = np.array([3, 4, 3, 4, 3, 4, 3, 4])\n", "spec_ac = snlay.calc_spectrum(size, mats, lams)\n", "\n", "print(size)\n", "\n", "\n", "size = (size - 50.0)/20.0\n", "\n", "\n", "\n", "spec = model.predict(np.expand_dims(size, axis = 0))\n", "\n", "spec = np.ravel(spec)\n", "\n", "\n", "fig1 = plt.figure(figsize=(11,3))\n", "gs = gridspec.GridSpec(1, 2, width_ratios=[8, 3]) \n", "\n", "ax = plt.subplot(gs[0])\n", "#ax = fig1.add_subplot(1,2,1)\n", "#ax.set_title('silica coated gold')\n", "ax.set_xlabel('Wavelength (nm)')\n", "ax.set_ylabel('Extinction Efficiency Qe')\n", "ax.set_ylim((0, 6))\n", "ax.set_xlim((300, 1200))\n", "plt.plot(lams, spec_ac,'b', linewidth=1, label='True')\n", "plt.plot(lams, spec, 'r--', linewidth=2, label='predicted')\n", "plt.plot(lams, 10*np.abs(spec_ac - spec)/spec_ac,'k', linewidth=1, label='10x Relative error')\n", "ax.legend(loc='best')\n", "\n", "ax2 = plt.subplot(gs[1])\n", "#fig2 = plt.figure(figsize=(3,3))\n", "#ax2 = fig1.add_subplot(1,2,2)\n", "#ax.set_title('silica coated gold')\n", "ax2.set_xlabel('Wavelength (nm)')\n", "#ax.set_ylabel('Extinction Efficiency Qe')\n", "#ax2.set_ylim((2, 6))\n", "ax2.set_xlim((300, 450))\n", "plt.plot(lams, spec_ac,'b', linewidth=1, label='True')\n", "plt.plot(lams, spec, 'r--', linewidth=2, label='predicted')\n", "#plt.plot(lams, 10*np.abs(spec_ac - spec)/spec_ac,'b', linewidth=1, label='10x Relative error')\n", "#ax.legend(loc='best')\n", "\n", "\n", "\n", "plt.tight_layout()\n", "plt.savefig('grid_figure.pdf')\n", "\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2018-09-30T12:48:05.013204Z", "start_time": "2018-09-30T12:48:04.790487Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.829695701599121\n" ] }, { "ename": "NameError", "evalue": "name 'result' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mNameError\u001b[0m: name 'result' is not defined" ] } ], "source": [ "# loop for hundred runs \n", "import snlay as snlay\n", "import time\n", "\n", "mats = np.array([3, 4, 3, 4, 3, 4, 3, 4])\n", "#spec_ac = snlay.calc_spectrum(size, mats, lams)\n", "\n", "reps = 100\n", "\n", "\n", "start = time.time()\n", "for ind in np.arange(reps):\n", " size = np.random.randint(30, 71, 8)\n", " #size = np.random.randint(30, 71, (1,8))\n", " #spec_ac = snlay.calc_spectrum(size, mats, lams)\n", " size = (size - 50.0)/20.0\n", " size = np.expand_dims(size, axis=0)\n", " spec = model.predict(size)\n", " #data_iter = mx.io.NDArrayIter(size, None, 1)\n", " #result = mod.predict(size)\n", " #result = result.asnumpy()\n", "\n", " \n", "end = time.time()\n", "print(1000*(end - start)/reps) \n", " \n", "\n", "\n", "result\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "### Inverse scattering " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-29T09:59:20.270577Z", "start_time": "2018-09-29T09:59:20.206497Z" }, "hidden": true }, "outputs": [], "source": [ "# from keras.models import Model\n", "# from keras.layers import Input, Add, AveragePooling1D, MaxPooling1D, Concatenate\n", "\n", "\n", "\n", "# a = Input(shape=(1,))\n", "# first = Dense(8, input_dim=1, kernel_initializer='normal', activation='linear', \n", "# name='dummy', use_bias=False)(a)\n", "# last = model(first)\n", "# #last = Dense(256, kernel_initializer='normal')(first)\n", "# #last = Flatten()(first)\n", "\n", "# model_d = Model(inputs=a, outputs=last)\n", "# model_d.compile(loss=naive_percent_loss, optimizer='nadam', metrics=[calc_mre_K], context=['gpu(0)'])\n", "\n", "model_d = Sequential()\n", "model_d.add(Dense(8, input_dim=1, kernel_initializer='normal', activation='linear', \n", " name='dummy', use_bias=False))\n", "\n", "for layer in model.layers[1:]:\n", " model_d.add(layer)\n", "\n", "# for layer in model_d.layers[1:]:\n", "# layer.trainable = False\n", "\n", "# for ind in range(1,len(model_d.layers)):\n", "# model_d.layers[ind].set_weights(model.layers[ind-1].get_weights())\n", "\n", "model_d.compile(loss=naive_percent_loss, optimizer='adam') \n", " \n", " \n", "model_d.summary()\n", "\n", "\n", "for layer in model.layers[1:]:\n", " print(layer)\n", "\n", "\n", "\n", "\n", "\n", "\n", "# # # let us create a target spectrum first\n", "# import snlay as snlay\n", "# #size = np.array([60, 65, 65, 65, 35, 35, 35, 35])\n", "# size = np.random.randint(30,70,8)\n", "# mats = np.array([3, 4, 3, 4, 3, 4, 3, 4])\n", "# target = snlay.calc_spectrum(size, mats, lams)\n", "\n", "# print(size)\n", "\n", "# # #do the training here\n", "# xd_t = np.ones((1,1))\n", "# yd_t = target.reshape(1,250)\n", "\n", "\n", "\n", "\n", "\n", "# history = model_d.fit(xd_t, yd_t,\n", "# batch_size=1,\n", "# epochs=5000, \n", "# verbose=0)\n", "\n", "# # #here is the final result\n", "# size_out = model_d.get_layer('dummy')\n", "# wts = size_out.get_weights()\n", "# wts = np.array(wts).ravel()\n", "# size_res= 0.5*wts*(size_max - size_min) + size_av\n", "# size_res_rounded = np.round(size_res)\n", "\n", "# spec_zer = model_d.predict(xd_t).ravel()\n", "# achieved = snlay.calc_spectrum(size_res_rounded, mats, lams)\n", "\n", "\n", "# fig1 = plt.figure(figsize=(22,5))\n", "# ax = fig1.add_subplot(1,1,1)\n", "# #plt.plot(lams, spec_zer, label='new model')\n", "# plt.plot(lams, target, linewidth=2, label='target')\n", "# plt.plot(lams, achieved, '--', linewidth=3, label='achieved')\n", "# plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n", "\n", "# print(size_res_rounded)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-26T10:28:37.316134Z", "start_time": "2018-09-26T10:27:22.339834Z" }, "hidden": true }, "outputs": [], "source": [ "from keras.models import Model\n", "from keras.layers import Input, Add, AveragePooling1D, MaxPooling1D, Concatenate\n", "\n", "a = Input(shape=(8,))\n", "first = Dense(256, kernel_initializer='normal')(a)\n", "#first = Dense(128, kernel_initializer='normal')(first)\n", "#first = BatchNormalization()(first)\n", "first= Activation('relu')(first)\n", "\n", "first = Reshape((256,1))(first)\n", "#first = UpSampling1D(size = 2)(first)\n", "first = Conv1D(filters=32, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal', activation='relu')(first)\n", "#first = UpSampling1D(size = 2)(first)\n", "\n", "\n", "\n", "\n", "# first_1 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", "# kernel_initializer='normal')(first)\n", "# first_1 = Activation('relu')(first_1)\n", "# first_1 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", "# kernel_initializer='normal')(first_1)\n", "first_2 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "#first_2 = Activation('relu')(first_2)\n", "first_2 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", " kernel_initializer='normal')(first_2)\n", "first_3 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "first_3 = Activation('relu')(first_3)\n", "first_3 = Conv1D(filters=8, kernel_size=5, strides=1, padding='same', \n", " kernel_initializer='normal')(first_3)\n", "first_4 = Conv1D(filters=16, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "list_of_tensors = [first_2, first_3, first_4]\n", "conc = Concatenate()(list_of_tensors)\n", "first = Add()([first, conc])\n", "first= Activation('relu')(first)\n", "\n", "\n", "\n", "# first_1 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", "# kernel_initializer='normal')(first)\n", "# first_1 = Activation('relu')(first_1)\n", "# first_1 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", "# kernel_initializer='normal')(first_1)\n", "first_2 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "#first_2 = Activation('relu')(first_2)\n", "first_2 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", " kernel_initializer='normal')(first_2)\n", "first_3 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "first_3 = Activation('relu')(first_3)\n", "first_3 = Conv1D(filters=8, kernel_size=5, strides=1, padding='same', \n", " kernel_initializer='normal')(first_3)\n", "first_4 = Conv1D(filters=16, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "list_of_tensors = [first_2, first_3, first_4]\n", "conc = Concatenate()(list_of_tensors)\n", "first = Add()([first, conc])\n", "first= Activation('relu')(first)\n", "\n", "\n", "\n", "\n", "# first_1 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", "# kernel_initializer='normal')(first)\n", "# first_1 = Activation('relu')(first_1)\n", "# first_1 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", "# kernel_initializer='normal')(first_1)\n", "first_2 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "#first_2 = Activation('relu')(first_2)\n", "first_2 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", " kernel_initializer='normal')(first_2)\n", "first_3 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "first_3 = Activation('relu')(first_3)\n", "first_3 = Conv1D(filters=8, kernel_size=5, strides=1, padding='same', \n", " kernel_initializer='normal')(first_3)\n", "first_4 = Conv1D(filters=16, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "list_of_tensors = [first_2, first_3, first_4]\n", "conc = Concatenate()(list_of_tensors)\n", "first = Add()([first, conc])\n", "first= Activation('relu')(first)\n", "\n", "# first_1 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", "# kernel_initializer='normal')(first)\n", "# first_1 = Activation('relu')(first_1)\n", "# first_1 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", "# kernel_initializer='normal')(first_1)\n", "first_2 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "#first_2 = Activation('relu')(first_2)\n", "first_2 = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', \n", " kernel_initializer='normal')(first_2)\n", "first_3 = Conv1D(filters=4, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "first_3 = Activation('relu')(first_3)\n", "first_3 = Conv1D(filters=8, kernel_size=5, strides=1, padding='same', \n", " kernel_initializer='normal')(first_3)\n", "first_4 = Conv1D(filters=16, kernel_size=1, strides=1, padding='same', \n", " kernel_initializer='normal')(first)\n", "list_of_tensors = [first_2, first_3, first_4]\n", "conc = Concatenate()(list_of_tensors)\n", "first = Add()([first, conc])\n", "first= Activation('relu')(first)\n", "\n", "\n", "\n", "\n", "\n", "first = Reshape((32,256))(first)\n", "first = MaxPooling1D(pool_size=32, strides=None, padding='same')(first)\n", "last = Flatten()(first)\n", "\n", "model = Model(inputs=a, outputs=last)\n", "model.compile(loss=naive_percent_loss, optimizer='nadam', metrics=[calc_mre_K], context=['gpu(0)'])\n", "\n", "#model.summary()\n", "\n", "from IPython.display import SVG\n", "from keras.utils.vis_utils import model_to_dot\n", "from keras.utils.vis_utils import plot_model\n", "\n", "#SVG(model_to_dot(model, show_shapes=True, show_layer_names=False).create(prog='dot', format='svg'))\n", "\n", "\n", "plot_model(model, show_shapes=False, show_layer_names=False, to_file='model.png')\n", "#plot_model(model, to_file='model.png', )\n", "\n", "x_t, x_v, y_t, y_v = train_test_split(x_train, y_train, test_size=0.2, random_state=42)\n", "# model = naiveploss_mgpu_model()\n", "# model.summary() \n", "history = model.fit(x_train, y_train,\n", " batch_size=64,\n", " epochs=2000, \n", " verbose=1,\n", " validation_data=(x_test, y_test))\n", "\n", "\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "### Model shipment" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-06T10:51:07.997455Z", "start_time": "2018-09-06T10:49:19.326Z" }, "hidden": true }, "outputs": [], "source": [ "\n", " \n", "from keras.models import load_model\n", "\n", "#Creates a HDF5 file 'my_model.h5'\n", "model.save('my_model.h5')\n", "\n", "# Deletes the existing model\n", "#del model \n", "\n", "# Returns a compiled model identical to the previous one\n", "#model = load_model('my_model.h5')\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-08T09:40:51.323101Z", "start_time": "2018-09-08T09:40:50.155098Z" }, "hidden": true }, "outputs": [], "source": [ "\n", "from keras.models import Model\n", "from keras.layers import Input, Add, Lambda, Dense\n", "import numpy as np\n", "\n", "\n", "def dropper(x):\n", " ms = 4\n", " #print(x.shape)\n", " \n", " \n", " \n", " return x**2\n", "# msk = np.array([1,1,1,1,0,0,0,0])\n", "\n", "\n", "a = Input(shape=(1,))\n", "b = Dense(8, input_dim=1, kernel_initializer='normal', activation='linear', \n", " name='dummy', use_bias=False)(a)\n", "b = Lambda(dropper)(b)\n", "b = Dense(256)(b)\n", "# #b = Lambda()\n", "# a = Lambda(dropper)(a)\n", "# \n", "\n", "model = Model(inputs=a, outputs=b)\n", "\n", "# model = Sequential()\n", "# model.add(Dense(256, input_dim=8))\n", "# #model.add(Lambda(lambda x: x**2))\n", "# model.add(Lambda(dropper))\n", "# model.add(Dense(256))\n", "\n", "\n", "\n", "model.compile(loss=naive_percent_loss, optimizer='adam', metrics=[calc_mre_K])\n", "#model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-28T10:32:16.469180Z", "start_time": "2018-09-28T10:32:16.452138Z" }, "hidden": true }, "outputs": [], "source": [ "2 +2 " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2018-09-28T10:32:36.778144Z", "start_time": "2018-09-28T10:32:36.682162Z" }, "hidden": true }, "outputs": [], "source": [ "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "hidden": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.6" } }, "nbformat": 4, "nbformat_minor": 2 }