commit with assisted DE

.ipynb_checkpoints/pygmoimp-checkpoint.ipynb

@@ -0,0 +1,240 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-03T07:12:19.852128Z",
+     "start_time": "2018-10-03T07:12:19.644845Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n",
+      "Dataset has been loaded\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import h5py\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "#import jtplot submodule from jupyterthemes\n",
+    "from jupyterthemes import jtplot\n",
+    "#currently installed theme will be used to\n",
+    "#set plot style if no arguments provided\n",
+    "jtplot.style()\n",
+    "\n",
+    "#now load this dataset \n",
+    "h5f = h5py.File('./datasets/s8_sio2tio2_v2.h5','r')\n",
+    "X = h5f['sizes'][:]\n",
+    "Y = h5f['spectrum'][:]\n",
+    "\n",
+    "#create a train - test split of the dataset\n",
+    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.4, random_state=42)\n",
+    "# normalize inputs \n",
+    "x_test = (x_test - 50)/20 \n",
+    "\n",
+    "print(\"Dataset has been loaded\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-03T07:12:22.108057Z",
+     "start_time": "2018-10-03T07:12:22.084707Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([0., 0., 0., 0., 0., 0., 0., 0.])"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import snlay as snlay\n",
+    "\n",
+    "np.zeros(8)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-03T07:12:26.001435Z",
+     "start_time": "2018-10-03T07:12:25.974477Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from PyGMO.problem import base\n",
+    "\n",
+    "import snlay as snlay\n",
+    "\n",
+    "mats = np.array([3, 4, 3, 4, 3, 4, 3, 4])\n",
+    "lams = np.linspace(300, 1200, 256)\n",
+    "center = np.zeros(8) + 57\n",
+    "targ_spec = y_test[19]\n",
+    "\n",
+    "def get_best_ans(islets):\n",
+    "    champ_fits = np.array([islets[idx].population.champion.f[0] for idx in range(len(islets))])\n",
+    "    minidx = np.argmin(champ_fits)\n",
+    "    midx = np.asscalar(minidx)\n",
+    "    best_x = np.array(archi[midx].population.champion.x)\n",
+    "    return best_x, np.amin(champ_fits)\n",
+    "\n",
+    "\n",
+    "def sphere_func(x, dims):\n",
+    "    x_np = np.array(x)\n",
+    "    spec_ac = snlay.calc_spectrum(x_np, mats, lams)\n",
+    "    diff = np.abs(targ_spec - spec_ac)/targ_spec\n",
+    "    #print(targ_spec.shape)\n",
+    "    #val = np.sum((x_np - center)**2)\n",
+    "    val = 100*np.mean(diff)\n",
+    "    return val\n",
+    "\n",
+    "\n",
+    "\n",
+    "class my_problem(base):\n",
+    "    \"\"\"\n",
+    "    \"\"\"\n",
+    "    def __init__(self, dim=8):\n",
+    "        # First we call the constructor of the base class telling PyGMO\n",
+    "        # what kind of problem to expect ('dim' dimensions, 1 objective, 0 contraints etc.)\n",
+    "        super(my_problem,self).__init__(dim)\n",
+    "         # We set the problem bounds (in this case equal for all components)\n",
+    "        self.set_bounds(30, 70)\n",
+    "     \n",
+    "\n",
+    "    # Reimplement the virtual method that defines the objective function.\n",
+    "    def _objfun_impl(self, x):\n",
+    "        # Compute the sphere function\n",
+    "        f = sphere_func(x,self.dimension)\n",
+    "        #f = sum([x[i] ** 2 for i in range(self.dimension)])\n",
+    "\n",
+    "        # Note that we return a tuple with one element only. In PyGMO the objective functions\n",
+    "        # return tuples so that multi-objective optimization is also possible.\n",
+    "        return (f, )\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-01T17:25:37.251830Z",
+     "start_time": "2018-10-01T17:25:37.204999Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "sphere_func((55, 65, 55, 65, 55, 65, 55, 65), 8)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "start_time": "2018-10-03T07:11:33.190Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import PyGMO as pyg\n",
+    "\n",
+    "prob = my_problem(dim=8)  # Create a 10-dimensional problem\n",
+    "\n",
+    "#algo = pyg.algorithm.bee_colony(gen=50)  # 500 generations of bee_colony algorithm\n",
+    "\n",
+    "# # # prob = pyg.problem.schwefel(dim = 8)\n",
+    "algo = pyg.algorithm.de(gen = 100)\n",
+    "archi = pyg.archipelago(algo,prob,8,20, topology = pyg.topology.ring())\n",
+    "\n",
+    "epocs = 10\n",
+    "es = np.arange(epocs)\n",
+    "opt_hist = []\n",
+    "for epch in range(epocs):\n",
+    "    archi.evolve(1)\n",
+    "    x, f = get_best_ans(archi)\n",
+    "    opt_hist.append(f)\n",
+    "    print(f)\n",
+    "\n",
+    "print(x)\n",
+    "\n",
+    "print(50 + 20*x_test[19])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-01T17:47:25.496340Z",
+     "start_time": "2018-10-01T17:47:25.472208Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "opt_hist"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig1 = plt.figure(figsize=(5,3))\n",
+    "ax = fig1.add_subplot(1,1,1)\n",
+    "#ax.set_title('silica coated gold')\n",
+    "ax.set_xlabel('Evolution')\n",
+    "ax.set_ylabel('Objective')\n",
+    "ax.semilogy(es, opt_hist)\n",
+    "ax.spines['top'].set_visible(False)\n",
+    "ax.spines['right'].set_visible(False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

.ipynb_checkpoints/scipydiffe-checkpoint.ipynb

@@ -0,0 +1,360 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T12:20:08.818657Z",
+     "start_time": "2018-10-05T12:20:08.561719Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n",
+      "Dataset has been loaded\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import h5py\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "#import jtplot submodule from jupyterthemes\n",
+    "from jupyterthemes import jtplot\n",
+    "#currently installed theme will be used to\n",
+    "#set plot style if no arguments provided\n",
+    "jtplot.style()\n",
+    "\n",
+    "#now load this dataset \n",
+    "h5f = h5py.File('./datasets/s8_sio2tio2_v2.h5','r')\n",
+    "X = h5f['sizes'][:]\n",
+    "Y = h5f['spectrum'][:]\n",
+    "\n",
+    "#create a train - test split of the dataset\n",
+    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.4, random_state=42)\n",
+    "# normalize inputs \n",
+    "#x_test = (x_test - 50)/20 \n",
+    "\n",
+    "print(\"Dataset has been loaded\")\n",
+    "\n",
+    "import numpy as np\n",
+    "import mxnet as mx\n",
+    "\n",
+    "# Step1: Load the model in MXNet\n",
+    "\n",
+    "# Use the same prefix and epoch parameters we used in save_mxnet_model API.\n",
+    "sym, arg_params, aux_params = mx.model.load_checkpoint(prefix='my_mod_bet', epoch=0)\n",
+    "\n",
+    "# We use the data_names and data_shapes returned by save_mxnet_model API.\n",
+    "mod = mx.mod.Module(symbol=sym, \n",
+    "                    data_names=['/input_21'], \n",
+    "                    context=mx.gpu(), \n",
+    "                    label_names=None)\n",
+    "mod.bind(for_training=False, \n",
+    "         data_shapes=[('/input_21', (1,8))], \n",
+    "         label_shapes=mod._label_shapes)\n",
+    "mod.set_params(arg_params, aux_params, allow_missing=True)    \n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 192,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T13:31:25.375799Z",
+     "start_time": "2018-10-05T13:31:25.342858Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "def de(fobj, bounds, mut=0.8, crossp=0.7, popsize=20, its=1000):\n",
+    "    dimensions = len(bounds)\n",
+    "    pop = np.random.rand(popsize, dimensions)\n",
+    "    min_b, max_b = np.asarray(bounds).T\n",
+    "    diff = np.fabs(min_b - max_b)\n",
+    "    pop_denorm = min_b + pop * diff\n",
+    "    fitness = np.asarray([fobj(ind) for ind in pop_denorm])\n",
+    "    best_idx = np.argmin(fitness)\n",
+    "    best = pop_denorm[best_idx]\n",
+    "    for i in range(its):\n",
+    "        #trialarr = np.zeros(popsize)\n",
+    "        for j in range(popsize):\n",
+    "            idxs = [idx for idx in range(popsize) if idx != j]\n",
+    "            a, b, c = pop[np.random.choice(idxs, 3, replace = False)]\n",
+    "            #mutant = np.clip(a + mut * (b - c), 0, 1)\n",
+    "            mutant = np.clip(pop[best_idx] + mut * (b - c), 0, 1)\n",
+    "            cross_points = np.random.rand(dimensions) < crossp\n",
+    "            if not np.any(cross_points):\n",
+    "                cross_points[np.random.randint(0, dimensions)] = True\n",
+    "            trial = np.where(cross_points, mutant, pop[j])\n",
+    "            #trialarr[j] = np.where(cross_points, mutant, pop[j])\n",
+    "            trial_denorm = min_b + trial * diff\n",
+    "            f = fobj(trial_denorm)\n",
+    "            if f < fitness[j]:\n",
+    "                fitness[j] = f\n",
+    "                pop[j] = trial\n",
+    "                if f < fitness[best_idx]:\n",
+    "                    best_idx = j\n",
+    "                    best = trial_denorm\n",
+    "        if i%25 == 0:\n",
+    "            print(i, fitness[best_idx])\n",
+    "    return best, fitness[best_idx]\n",
+    "\n",
+    "\n",
+    "def de2(fobj, bounds, mut=0.8, crossp=0.7, popsize=20, its=1000):\n",
+    "    dimensions = len(bounds)\n",
+    "    pop = np.random.rand(popsize, dimensions)\n",
+    "    min_b, max_b = np.asarray(bounds).T\n",
+    "    diff = np.fabs(min_b - max_b)\n",
+    "    pop_denorm = min_b + pop * diff\n",
+    "    fitness = np.asarray(fobj(pop_denorm))\n",
+    "    best_idx = np.argmin(fitness)\n",
+    "    best = pop_denorm[best_idx]\n",
+    "    for i in range(its):\n",
+    "        trialarr = np.zeros((popsize, dimensions))\n",
+    "        for j in range(popsize):\n",
+    "            idxs = [idx for idx in range(popsize) if idx != j]\n",
+    "            a, b, c = pop[np.random.choice(idxs, 3, replace = False)]\n",
+    "            #mutant = np.clip(a + mut * (b - c), 0, 1)\n",
+    "            mutant = np.clip(pop[best_idx] + mut * (b - c), 0, 1)\n",
+    "            cross_points = np.random.rand(dimensions) < crossp\n",
+    "            if not np.any(cross_points):\n",
+    "                cross_points[np.random.randint(0, dimensions)] = True\n",
+    "            trialarr[j] = np.where(cross_points, mutant, pop[j])\n",
+    "            \n",
+    "        \n",
+    "        trial_denorm = min_b + trialarr * diff\n",
+    "        #print(trial_denorm)\n",
+    "        f = fobj(trial_denorm)\n",
+    "        for j in range(popsize):\n",
+    "            if f[j] < fitness[j]:\n",
+    "                fitness[j] = f[j]\n",
+    "                pop[j] = trialarr[j]\n",
+    "                if f[j] < fitness[best_idx]:\n",
+    "                    best_idx = j\n",
+    "                    best = trial_denorm[j]\n",
+    "        if i%25 == 0:\n",
+    "            print(i, fitness[best_idx])\n",
+    "    return best, fitness[best_idx]\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "start_time": "2018-10-05T13:30:27.166Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0 1.9676681607961655\n",
+      "25 0.7767577655613422\n",
+      "50 0.7767577655613422\n",
+      "75 0.7767577655613422\n",
+      "100 0.7324242033064365\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import numpy as np\n",
+    "from scipy.optimize import rosen, differential_evolution\n",
+    "import scnets as scn\n",
+    "import mxnet as mx\n",
+    "from mxnet import nd\n",
+    "import snlay as snlay\n",
+    "\n",
+    "mats = np.array([3, 4, 3, 4, 3, 4, 3, 4])\n",
+    "lams = np.linspace(300, 1200, 256)\n",
+    "targ_spec = y_test[29]\n",
+    "targ_spec2 = np.tile(targ_spec, (80,1))\n",
+    "\n",
+    "# def loss_jumper(x, it_cnt=0):\n",
+    "\n",
+    "def mxmod_arr_loss(x):\n",
+    "    x_np = np.array(x)\n",
+    "    x_np = (x_np - 50.0)/20.0\n",
+    "    res2 = mod.predict(x_np)\n",
+    "    y_t = nd.array(targ_spec2, ctx=mx.gpu())\n",
+    "    err = nd.abs(y_t - res2)/y_t\n",
+    "    err2 = 100*nd.mean(err, axis=1).asnumpy()\n",
+    "    return err2\n",
+    "\n",
+    "def mxmod_loss(x):\n",
+    "    x_np = np.array(x)\n",
+    "    x_np = (x_np - 50.0)/20.0\n",
+    "    x_np = np.expand_dims(x_np, axis=0)\n",
+    "    res2 = mod.predict(x_np)\n",
+    "    y_t = nd.array(targ_spec, ctx=mx.gpu())\n",
+    "    err = nd.abs(y_t - res2)/y_t\n",
+    "    err2 = 100*nd.mean(err).asscalar()\n",
+    "    return err2\n",
+    "\n",
+    "#     res2 = mod.predict(x)\n",
+    "\n",
+    "#     \n",
+    "#     #ez2 = err2.copyto(mx.cpu())\n",
+    "#     return err2\n",
+    "\n",
+    "def loss_func(x):\n",
+    "    #x_np = np.array(x)\n",
+    "    #count+=1\n",
+    "    spec_ac = snlay.calc_spectrum(x, mats, lams)\n",
+    "    diff = np.abs(targ_spec - spec_ac)/targ_spec\n",
+    "    return 100*np.mean(diff)\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "# x_t = np.array([40, 45, 40, 45, 40, 45, 40, 50])\n",
+    "# sd = mxmod_loss(x_t)\n",
+    "# sd2 = loss_func(x_t)\n",
+    "# print(sd, sd2)\n",
+    "bnds = [(30, 70)]*8\n",
+    "#b, c = de(fobj=loss_func, bounds=bnds, popsize=80, its=2500)\n",
+    "b, c = de(fobj=mxmod_loss, bounds=bnds, popsize=80, its=2500)\n",
+    "#b, c = de2(fobj=mxmod_arr_loss, bounds=bnds, popsize=80, its=2500)                 \n",
+    "#it = list(de(lambda x: x**2, bounds=[(-100, 100)]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 121,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T13:09:02.641621Z",
+     "start_time": "2018-10-05T13:09:02.616461Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(80, 256)"
+      ]
+     },
+     "execution_count": 121,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "targ_spec.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 105,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T12:35:59.739963Z",
+     "start_time": "2018-10-05T12:35:59.714914Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([30., 64., 52., 36., 36., 64., 35., 62.])"
+      ]
+     },
+     "execution_count": 105,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x_test[29]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 114,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T13:04:11.358640Z",
+     "start_time": "2018-10-05T13:04:11.334873Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "a = np.array([0, 1, 2])\n",
+    "b = np.tile(a, (3,1))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 115,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T13:04:13.495202Z",
+     "start_time": "2018-10-05T13:04:13.469641Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(3, 3)"
+      ]
+     },
+     "execution_count": 115,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "b.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

hypoptim.ipynb

@@ -344,7 +344,25 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.6"
+   "version": "3.5.4"
+  },
+  "latex_envs": {
+   "LaTeX_envs_menu_present": true,
+   "autoclose": false,
+   "autocomplete": true,
+   "bibliofile": "biblio.bib",
+   "cite_by": "apalike",
+   "current_citInitial": 1,
+   "eqLabelWithNumbers": true,
+   "eqNumInitial": 1,
+   "hotkeys": {
+    "equation": "Ctrl-E",
+    "itemize": "Ctrl-I"
+   },
+   "labels_anchors": false,
+   "latex_user_defs": false,
+   "report_style_numbering": false,
+   "user_envs_cfg": false
   }
  },
  "nbformat": 4,

models/my_mod_bet-symbol.json

@@ -0,0 +1,678 @@
+{
+  "nodes": [
+    {
+      "op": "null", 
+      "name": "/input_21", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(0, 8)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "dense_2/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot11", 
+      "inputs": [[0, 0, 0], [1, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "dense_2/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add25", 
+      "inputs": [[2, 0, 0], [3, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu15", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[4, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_2/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar13", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[6, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar15", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[4, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu17", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[8, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul7", 
+      "inputs": [[7, 0, 0], [9, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add27", 
+      "inputs": [[5, 0, 0], [10, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape15", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[11, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims17", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[12, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose27", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[13, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_5/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims19", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[15, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose28", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[16, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "conv1d_5/conv1d2", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[14, 0, 0], [17, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose29", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[18, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape17", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[19, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_5/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add29", 
+      "inputs": [[20, 0, 0], [21, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu19", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[22, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims21", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[23, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose33", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[24, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_6/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims23", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[26, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose34", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[27, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "conv1d_6/conv1d2", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[25, 0, 0], [28, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose35", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[29, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape19", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[30, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_6/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add31", 
+      "inputs": [[31, 0, 0], [32, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add33", 
+      "inputs": [[33, 0, 0], [12, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims25", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[34, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose39", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[35, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_7/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims27", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[37, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose40", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[38, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "conv1d_7/conv1d2", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[36, 0, 0], [39, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose41", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[40, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape21", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[41, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_7/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add35", 
+      "inputs": [[42, 0, 0], [43, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu21", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[44, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims29", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[45, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose45", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[46, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_8/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims31", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[48, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose46", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[49, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "conv1d_8/conv1d2", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[47, 0, 0], [50, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose47", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[51, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape23", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[52, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_8/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add37", 
+      "inputs": [[53, 0, 0], [54, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add39", 
+      "inputs": [[55, 0, 0], [34, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims33", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[56, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose51", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[57, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_9/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims35", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[59, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose52", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[60, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "conv1d_9/conv1d2", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[58, 0, 0], [61, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose53", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[62, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape25", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[63, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_9/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add41", 
+      "inputs": [[64, 0, 0], [65, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu23", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[66, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims37", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[67, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose57", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[68, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_10/kernel1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims39", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[70, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose58", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[71, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "conv1d_10/conv1d2", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[69, 0, 0], [72, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose59", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[73, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape27", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[74, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "conv1d_10/bias1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add43", 
+      "inputs": [[75, 0, 0], [76, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add45", 
+      "inputs": [[77, 0, 0], [56, 0, 0]]
+    }, 
+    {
+      "op": "Flatten", 
+      "name": "flatten3", 
+      "inputs": [[78, 0, 0]]
+    }
+  ], 
+  "arg_nodes": [
+    0, 
+    1, 
+    3, 
+    6, 
+    15, 
+    21, 
+    26, 
+    32, 
+    37, 
+    43, 
+    48, 
+    54, 
+    59, 
+    65, 
+    70, 
+    76
+  ], 
+  "node_row_ptr": [
+    0, 
+    1, 
+    2, 
+    3, 
+    4, 
+    5, 
+    6, 
+    7, 
+    8, 
+    9, 
+    10, 
+    11, 
+    12, 
+    13, 
+    14, 
+    15, 
+    16, 
+    17, 
+    18, 
+    19, 
+    20, 
+    21, 
+    22, 
+    23, 
+    24, 
+    25, 
+    26, 
+    27, 
+    28, 
+    29, 
+    30, 
+    31, 
+    32, 
+    33, 
+    34, 
+    35, 
+    36, 
+    37, 
+    38, 
+    39, 
+    40, 
+    41, 
+    42, 
+    43, 
+    44, 
+    45, 
+    46, 
+    47, 
+    48, 
+    49, 
+    50, 
+    51, 
+    52, 
+    53, 
+    54, 
+    55, 
+    56, 
+    57, 
+    58, 
+    59, 
+    60, 
+    61, 
+    62, 
+    63, 
+    64, 
+    65, 
+    66, 
+    67, 
+    68, 
+    69, 
+    70, 
+    71, 
+    72, 
+    73, 
+    74, 
+    75, 
+    76, 
+    77, 
+    78, 
+    79, 
+    80
+  ], 
+  "heads": [[79, 0, 0]], 
+  "attrs": {"mxnet_version": ["int", 10300]}
+}

models/my_mod_conv1d-symbol.json

@@ -0,0 +1,559 @@
+{
+  "nodes": [
+    {
+      "op": "null", 
+      "name": "/first_input4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(0, 8)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "first/kernel4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot23", 
+      "inputs": [[0, 0, 0], [1, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "first/bias4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add67", 
+      "inputs": [[2, 0, 0], [3, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu43", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[4, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape47", 
+      "attrs": {"shape": "(0, 4, 64)"}, 
+      "inputs": [[5, 0, 0]]
+    }, 
+    {
+      "op": "repeat", 
+      "name": "repeat3", 
+      "attrs": {
+        "axis": "1", 
+        "repeats": "2"
+      }, 
+      "inputs": [[6, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims61", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[7, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose93", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[8, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv1/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(5, 64, 64)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims63", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[10, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose94", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[11, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv1/conv1d4", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(5, 1)", 
+        "no_bias": "True", 
+        "num_filter": "64", 
+        "pad": "(2, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[9, 0, 0], [12, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose95", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[13, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape49", 
+      "attrs": {"shape": "(0, 8, 64)"}, 
+      "inputs": [[14, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv1/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(64,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add69", 
+      "inputs": [[15, 0, 0], [16, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu45", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[17, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims65", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[18, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose99", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[19, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv2/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(5, 64, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims67", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[21, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose100", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[22, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv2/conv1d4", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(5, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(2, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[20, 0, 0], [23, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose101", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[24, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape51", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[25, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv2/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add71", 
+      "inputs": [[26, 0, 0], [27, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu47", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[28, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims69", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[29, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose105", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[30, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv3/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(5, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims71", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[32, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose106", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[33, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv3/conv1d4", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(5, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(2, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[31, 0, 0], [34, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose107", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[35, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape53", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[36, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv3/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add73", 
+      "inputs": [[37, 0, 0], [38, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu49", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[39, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims73", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[40, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose111", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[41, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv4/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(5, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims75", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[43, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose112", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[44, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv4/conv1d4", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(5, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(2, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[42, 0, 0], [45, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose113", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[46, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape55", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[47, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv4/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add75", 
+      "inputs": [[48, 0, 0], [49, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu51", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[50, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims77", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[51, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose117", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[52, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv5/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(5, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims79", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[54, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose118", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[55, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv5/conv1d4", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(5, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(2, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[53, 0, 0], [56, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose119", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[57, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape57", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[58, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv5/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add77", 
+      "inputs": [[59, 0, 0], [60, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu53", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[61, 0, 0]]
+    }, 
+    {
+      "op": "Flatten", 
+      "name": "flatten7", 
+      "inputs": [[62, 0, 0]]
+    }
+  ], 
+  "arg_nodes": [
+    0, 
+    1, 
+    3, 
+    10, 
+    16, 
+    21, 
+    27, 
+    32, 
+    38, 
+    43, 
+    49, 
+    54, 
+    60
+  ], 
+  "node_row_ptr": [
+    0, 
+    1, 
+    2, 
+    3, 
+    4, 
+    5, 
+    6, 
+    7, 
+    8, 
+    9, 
+    10, 
+    11, 
+    12, 
+    13, 
+    14, 
+    15, 
+    16, 
+    17, 
+    18, 
+    19, 
+    20, 
+    21, 
+    22, 
+    23, 
+    24, 
+    25, 
+    26, 
+    27, 
+    28, 
+    29, 
+    30, 
+    31, 
+    32, 
+    33, 
+    34, 
+    35, 
+    36, 
+    37, 
+    38, 
+    39, 
+    40, 
+    41, 
+    42, 
+    43, 
+    44, 
+    45, 
+    46, 
+    47, 
+    48, 
+    49, 
+    50, 
+    51, 
+    52, 
+    53, 
+    54, 
+    55, 
+    56, 
+    57, 
+    58, 
+    59, 
+    60, 
+    61, 
+    62, 
+    63, 
+    64
+  ], 
+  "heads": [[63, 0, 0]], 
+  "attrs": {"mxnet_version": ["int", 10300]}
+}

models/my_mod_convprel-symbol.json

@@ -0,0 +1,841 @@
+{
+  "nodes": [
+    {
+      "op": "null", 
+      "name": "/first_input6", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(0, 8)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "first/kernel6", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot27", 
+      "inputs": [[0, 0, 0], [1, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "first/bias6", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add103", 
+      "inputs": [[2, 0, 0], [3, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu79", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[4, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_9/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar61", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[6, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar63", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[4, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu81", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[8, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul31", 
+      "inputs": [[7, 0, 0], [9, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add105", 
+      "inputs": [[5, 0, 0], [10, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape75", 
+      "attrs": {"shape": "(0, 4, 64)"}, 
+      "inputs": [[11, 0, 0]]
+    }, 
+    {
+      "op": "repeat", 
+      "name": "repeat7", 
+      "attrs": {
+        "axis": "1", 
+        "repeats": "2"
+      }, 
+      "inputs": [[12, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims101", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[13, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose153", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[14, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv1/kernel4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 64, 64)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims103", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[16, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose154", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[17, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv1/conv1d8", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "64", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[15, 0, 0], [18, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose155", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[19, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape77", 
+      "attrs": {"shape": "(0, 8, 64)"}, 
+      "inputs": [[20, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv1/bias4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(64,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add107", 
+      "inputs": [[21, 0, 0], [22, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu83", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[23, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_10/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 64)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar65", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[25, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar67", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[23, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu85", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[27, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul33", 
+      "inputs": [[26, 0, 0], [28, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add109", 
+      "inputs": [[24, 0, 0], [29, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims105", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[30, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose159", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[31, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv2/kernel4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 64, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims107", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[33, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose160", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[34, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv2/conv1d8", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[32, 0, 0], [35, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose161", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[36, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape79", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[37, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv2/bias4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add111", 
+      "inputs": [[38, 0, 0], [39, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu87", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[40, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_11/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar69", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[42, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar71", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[40, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu89", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[44, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul35", 
+      "inputs": [[43, 0, 0], [45, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add113", 
+      "inputs": [[41, 0, 0], [46, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims109", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[47, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose165", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[48, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv3/kernel4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims111", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[50, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose166", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[51, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv3/conv1d8", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[49, 0, 0], [52, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose167", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[53, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape81", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[54, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv3/bias4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add115", 
+      "inputs": [[55, 0, 0], [56, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu91", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[57, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_12/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar73", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[59, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar75", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[57, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu93", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[61, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul37", 
+      "inputs": [[60, 0, 0], [62, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add117", 
+      "inputs": [[58, 0, 0], [63, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims113", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[64, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose171", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[65, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv4/kernel4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims115", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[67, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose172", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[68, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv4/conv1d8", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[66, 0, 0], [69, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose173", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[70, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape83", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[71, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv4/bias4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add119", 
+      "inputs": [[72, 0, 0], [73, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu95", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[74, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_13/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar77", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[76, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar79", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[74, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu97", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[78, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul39", 
+      "inputs": [[77, 0, 0], [79, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add121", 
+      "inputs": [[75, 0, 0], [80, 0, 0]]
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims117", 
+      "attrs": {"axis": "2"}, 
+      "inputs": [[81, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose177", 
+      "attrs": {"axes": "[0, 3, 1, 2]"}, 
+      "inputs": [[82, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv5/kernel4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(3, 32, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "expand_dims", 
+      "name": "expand_dims119", 
+      "attrs": {"axis": "1"}, 
+      "inputs": [[84, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose178", 
+      "attrs": {"axes": "(3, 2, 0, 1)"}, 
+      "inputs": [[85, 0, 0]]
+    }, 
+    {
+      "op": "Convolution", 
+      "name": "Conv5/conv1d8", 
+      "attrs": {
+        "dilate": "(1, 1)", 
+        "kernel": "(3, 1)", 
+        "no_bias": "True", 
+        "num_filter": "32", 
+        "pad": "(1, 0)", 
+        "stride": "(1, 1)"
+      }, 
+      "inputs": [[83, 0, 0], [86, 0, 0]]
+    }, 
+    {
+      "op": "transpose", 
+      "name": "transpose179", 
+      "attrs": {"axes": "[0, 2, 3, 1]"}, 
+      "inputs": [[87, 0, 0]]
+    }, 
+    {
+      "op": "Reshape", 
+      "name": "reshape85", 
+      "attrs": {"shape": "(0, 8, 32)"}, 
+      "inputs": [[88, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "Conv5/bias4", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(32,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add123", 
+      "inputs": [[89, 0, 0], [90, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu99", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[91, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "p_re_lu_14/alpha1", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 32)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar81", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[93, 0, 0]]
+    }, 
+    {
+      "op": "_mul_scalar", 
+      "name": "_mulscalar83", 
+      "attrs": {"scalar": "-1.0"}, 
+      "inputs": [[91, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu101", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[95, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_mul", 
+      "name": "broadcast_mul41", 
+      "inputs": [[94, 0, 0], [96, 0, 0]]
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add125", 
+      "inputs": [[92, 0, 0], [97, 0, 0]]
+    }, 
+    {
+      "op": "Flatten", 
+      "name": "flatten11", 
+      "inputs": [[98, 0, 0]]
+    }
+  ], 
+  "arg_nodes": [
+    0, 
+    1, 
+    3, 
+    6, 
+    16, 
+    22, 
+    25, 
+    33, 
+    39, 
+    42, 
+    50, 
+    56, 
+    59, 
+    67, 
+    73, 
+    76, 
+    84, 
+    90, 
+    93
+  ], 
+  "node_row_ptr": [
+    0, 
+    1, 
+    2, 
+    3, 
+    4, 
+    5, 
+    6, 
+    7, 
+    8, 
+    9, 
+    10, 
+    11, 
+    12, 
+    13, 
+    14, 
+    15, 
+    16, 
+    17, 
+    18, 
+    19, 
+    20, 
+    21, 
+    22, 
+    23, 
+    24, 
+    25, 
+    26, 
+    27, 
+    28, 
+    29, 
+    30, 
+    31, 
+    32, 
+    33, 
+    34, 
+    35, 
+    36, 
+    37, 
+    38, 
+    39, 
+    40, 
+    41, 
+    42, 
+    43, 
+    44, 
+    45, 
+    46, 
+    47, 
+    48, 
+    49, 
+    50, 
+    51, 
+    52, 
+    53, 
+    54, 
+    55, 
+    56, 
+    57, 
+    58, 
+    59, 
+    60, 
+    61, 
+    62, 
+    63, 
+    64, 
+    65, 
+    66, 
+    67, 
+    68, 
+    69, 
+    70, 
+    71, 
+    72, 
+    73, 
+    74, 
+    75, 
+    76, 
+    77, 
+    78, 
+    79, 
+    80, 
+    81, 
+    82, 
+    83, 
+    84, 
+    85, 
+    86, 
+    87, 
+    88, 
+    89, 
+    90, 
+    91, 
+    92, 
+    93, 
+    94, 
+    95, 
+    96, 
+    97, 
+    98, 
+    99, 
+    100
+  ], 
+  "heads": [[99, 0, 0]], 
+  "attrs": {"mxnet_version": ["int", 10300]}
+}

models/my_mod_fullycon-symbol.json

@@ -0,0 +1,178 @@
+{
+  "nodes": [
+    {
+      "op": "null", 
+      "name": "/first_input2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(0, 8)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "null", 
+      "name": "first/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(8, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot13", 
+      "inputs": [[0, 0, 0], [1, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "first/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add47", 
+      "inputs": [[2, 0, 0], [3, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu25", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[4, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "H0/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot15", 
+      "inputs": [[5, 0, 0], [6, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "H0/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add49", 
+      "inputs": [[7, 0, 0], [8, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu27", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[9, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "H1/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot17", 
+      "inputs": [[10, 0, 0], [11, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "H1/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add51", 
+      "inputs": [[12, 0, 0], [13, 0, 0]]
+    }, 
+    {
+      "op": "LeakyReLU", 
+      "name": "leakyrelu29", 
+      "attrs": {
+        "act_type": "leaky", 
+        "slope": "0.0"
+      }, 
+      "inputs": [[14, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "last/kernel2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256, 256)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "dot", 
+      "name": "dot19", 
+      "inputs": [[15, 0, 0], [16, 0, 0]]
+    }, 
+    {
+      "op": "null", 
+      "name": "last/bias2", 
+      "attrs": {
+        "__dtype__": "0", 
+        "__shape__": "(256,)"
+      }, 
+      "inputs": []
+    }, 
+    {
+      "op": "broadcast_add", 
+      "name": "broadcast_add53", 
+      "inputs": [[17, 0, 0], [18, 0, 0]]
+    }
+  ], 
+  "arg_nodes": [0, 1, 3, 6, 8, 11, 13, 16, 18], 
+  "node_row_ptr": [
+    0, 
+    1, 
+    2, 
+    3, 
+    4, 
+    5, 
+    6, 
+    7, 
+    8, 
+    9, 
+    10, 
+    11, 
+    12, 
+    13, 
+    14, 
+    15, 
+    16, 
+    17, 
+    18, 
+    19, 
+    20
+  ], 
+  "heads": [[19, 0, 0]], 
+  "attrs": {"mxnet_version": ["int", 10300]}
+}

mxmodel.ipynb

@@ -2,21 +2,21 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 21,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-09-30T16:20:36.613702Z",
-     "start_time": "2018-09-30T16:20:35.532595Z"
+     "end_time": "2018-10-03T13:53:26.552385Z",
+     "start_time": "2018-10-03T13:53:26.342168Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "array([67., 31., 30., 37., 56., 37., 61., 49.])"
+       "array([ 0.85, -0.95, -1.  , -0.65,  0.3 , -0.65,  0.55, -0.05])"
       ]
      },
-     "execution_count": 1,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -34,7 +34,7 @@
     "#create a train - test split of the dataset\n",
     "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.4, random_state=42)\n",
     "\n",
-    "\n",
+    "x_test = (x_test - 50.0)/20.0\n",
     "\n",
     "x_test[9]\n"
    ]
@@ -44,8 +44,8 @@
    "execution_count": 2,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-09-30T16:20:49.933501Z",
-     "start_time": "2018-09-30T16:20:45.837033Z"
+     "end_time": "2018-10-03T13:41:28.745978Z",
+     "start_time": "2018-10-03T13:41:26.725681Z"
     }
    },
    "outputs": [],
@@ -71,28 +71,65 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
+   "execution_count": 28,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-03T15:57:53.584259Z",
+     "start_time": "2018-10-03T15:57:53.321389Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "255.54943084716797\n"
+     ]
+    }
+   ],
    "source": [
-    "\n"
+    "import scnets as scn\n",
+    "import mxnet as mx\n",
+    "from mxnet import nd\n",
+    "import time\n",
+    "\n",
+    "\n",
+    "bsize = 4000\n",
+    "y_true = nd.array(y_test[9], ctx=mx.gpu())\n",
+    "reps = 1000\n",
+    "\n",
+    "start = time.time()\n",
+    "for icv in np.arange(reps):\n",
+    "    x_t = x_test[0:bsize]\n",
+    "    res2 = mod.predict(x_t)\n",
+    "    err = nd.abs(y_true - res2)/y_true\n",
+    "    finerr = 100*nd.mean(err)\n",
+    "end = time.time()\n",
+    "print(1000*(end - start))   "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-09-30T16:21:01.661357Z",
-     "start_time": "2018-09-30T16:21:01.485033Z"
+     "end_time": "2018-10-03T08:45:32.141435Z",
+     "start_time": "2018-10-03T08:45:31.874129Z"
     }
    },
    "outputs": [
     {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using MXNet backend\n"
+     ]
+    },
+    {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.16714763641357422\n"
+      "0.17115497589111328\n"
      ]
     }
    ],
@@ -127,11 +164,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-09-30T16:21:08.815235Z",
-     "start_time": "2018-09-30T16:21:08.809272Z"
+     "end_time": "2018-10-03T08:45:41.210914Z",
+     "start_time": "2018-10-03T08:45:41.205195Z"
     }
    },
    "outputs": [],
@@ -152,11 +189,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-09-30T16:49:04.385979Z",
-     "start_time": "2018-09-30T16:49:02.703408Z"
+     "end_time": "2018-10-03T08:45:59.234753Z",
+     "start_time": "2018-10-03T08:45:57.287759Z"
     }
    },
    "outputs": [
@@ -164,7 +201,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "1.672537088394165\n"
+      "1.9373879432678223\n"
      ]
     }
    ],
@@ -189,21 +226,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-09-30T16:21:45.903809Z",
-     "start_time": "2018-09-30T16:21:45.898205Z"
+     "end_time": "2018-10-03T08:45:54.615638Z",
+     "start_time": "2018-10-03T08:45:54.610110Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "248.17133"
+       "235.14212"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -261,7 +298,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.6"
+   "version": "3.5.4"
   }
  },
  "nbformat": 4,

pymietest.ipynb

@@ -144,8 +144,11 @@
     "# ./materials/silica.dat\n",
     "#I. H. Malitson. Interspecimen comparison of the refractive index of fused silica, J. Opt. Soc. Am. 55, 1205-1208 (1965)\n",
     "\n",
-    "# ./materials/tio2.dat\n",
-    "#T. Siefke, S. Kroker, K. Pfeiffer, O. Puffky, K. Dietrich, D. Franta, I. Ohlídal, A. Szeghalmi, E.-B. Kley, A. Tünnermann. Materials pushing the application limits of wire grid polarizers further into the deep ultraviolet spectral range, Adv. Opt. Mater. 4, 1780–1786 (2016)\n",
+    "# # ./materials/tio2.dat\n",
+    "# #T. Siefke, S. Kroker, K. Pfeiffer, O. Puffky, K. Dietrich, D. Franta, \n",
+    "# I. Ohlídal, A. Szeghalmi, E.-B. Kley, A. Tünnermann.\n",
+    "# Materials pushing the application limits of wire grid polarizers further into the deep\n",
+    "# ultraviolet spectral range, Adv. Opt. Mater. 4, 1780–1786 (2016)\n",
     "\n",
     "\n",
     "from scipy import interpolate\n",
@@ -696,7 +699,25 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.6"
+   "version": "3.5.4"
+  },
+  "latex_envs": {
+   "LaTeX_envs_menu_present": true,
+   "autoclose": false,
+   "autocomplete": true,
+   "bibliofile": "biblio.bib",
+   "cite_by": "apalike",
+   "current_citInitial": 1,
+   "eqLabelWithNumbers": true,
+   "eqNumInitial": 1,
+   "hotkeys": {
+    "equation": "Ctrl-E",
+    "itemize": "Ctrl-I"
+   },
+   "labels_anchors": false,
+   "latex_user_defs": false,
+   "report_style_numbering": false,
+   "user_envs_cfg": false
   }
  },
  "nbformat": 4,

results/model_resnet.svg

@@ -4,143 +4,188 @@
 <!-- Generated by graphviz version 2.38.0 (20140413.2041)
  -->
 <!-- Title: G Pages: 1 -->
-<svg width="120pt" height="921pt"
- viewBox="0.00 0.00 119.50 921.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 917)">
+<svg width="121pt" height="1213pt"
+ viewBox="0.00 0.00 120.50 1213.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1209)">
 <title>G</title>
-<polygon fill="white" stroke="none" points="-4,4 -4,-917 115.5,-917 115.5,4 -4,4"/>
-<!-- 140387210274520 -->
-<g id="node1" class="node"><title>140387210274520</title>
-<polygon fill="none" stroke="black" points="34.5,-876.5 34.5,-912.5 111.5,-912.5 111.5,-876.5 34.5,-876.5"/>
-<text text-anchor="middle" x="73" y="-890.8" font-family="Times,serif" font-size="14.00">InputLayer</text>
-</g>
-<!-- 140387210275360 -->
-<g id="node2" class="node"><title>140387210275360</title>
-<polygon fill="none" stroke="black" points="46,-803.5 46,-839.5 100,-839.5 100,-803.5 46,-803.5"/>
-<text text-anchor="middle" x="73" y="-817.8" font-family="Times,serif" font-size="14.00">Dense</text>
-</g>
-<!-- 140387210274520&#45;&gt;140387210275360 -->
-<g id="edge1" class="edge"><title>140387210274520&#45;&gt;140387210275360</title>
-<path fill="none" stroke="black" d="M73,-876.313C73,-868.289 73,-858.547 73,-849.569"/>
-<polygon fill="black" stroke="black" points="76.5001,-849.529 73,-839.529 69.5001,-849.529 76.5001,-849.529"/>
-</g>
-<!-- 140387210275528 -->
-<g id="node3" class="node"><title>140387210275528</title>
-<polygon fill="none" stroke="black" points="44.5,-730.5 44.5,-766.5 101.5,-766.5 101.5,-730.5 44.5,-730.5"/>
-<text text-anchor="middle" x="73" y="-744.8" font-family="Times,serif" font-size="14.00">PReLU</text>
-</g>
-<!-- 140387210275360&#45;&gt;140387210275528 -->
-<g id="edge2" class="edge"><title>140387210275360&#45;&gt;140387210275528</title>
-<path fill="none" stroke="black" d="M73,-803.313C73,-795.289 73,-785.547 73,-776.569"/>
-<polygon fill="black" stroke="black" points="76.5001,-776.529 73,-766.529 69.5001,-776.529 76.5001,-776.529"/>
-</g>
-<!-- 140387210396840 -->
-<g id="node4" class="node"><title>140387210396840</title>
-<polygon fill="none" stroke="black" points="42,-657.5 42,-693.5 104,-693.5 104,-657.5 42,-657.5"/>
-<text text-anchor="middle" x="73" y="-671.8" font-family="Times,serif" font-size="14.00">Reshape</text>
-</g>
-<!-- 140387210275528&#45;&gt;140387210396840 -->
-<g id="edge3" class="edge"><title>140387210275528&#45;&gt;140387210396840</title>
-<path fill="none" stroke="black" d="M73,-730.313C73,-722.289 73,-712.547 73,-703.569"/>
-<polygon fill="black" stroke="black" points="76.5001,-703.529 73,-693.529 69.5001,-703.529 76.5001,-703.529"/>
-</g>
-<!-- 140387210276368 -->
-<g id="node5" class="node"><title>140387210276368</title>
+<polygon fill="white" stroke="none" points="-4,4 -4,-1209 116.5,-1209 116.5,4 -4,4"/>
+<!-- 139802249115240 -->
+<g id="node1" class="node"><title>139802249115240</title>
+<polygon fill="none" stroke="black" points="35.5,-1168.5 35.5,-1204.5 112.5,-1204.5 112.5,-1168.5 35.5,-1168.5"/>
+<text text-anchor="middle" x="74" y="-1182.8" font-family="Times,serif" font-size="14.00">InputLayer</text>
+</g>
+<!-- 139802249116752 -->
+<g id="node2" class="node"><title>139802249116752</title>
+<polygon fill="none" stroke="black" points="47,-1095.5 47,-1131.5 101,-1131.5 101,-1095.5 47,-1095.5"/>
+<text text-anchor="middle" x="74" y="-1109.8" font-family="Times,serif" font-size="14.00">Dense</text>
+</g>
+<!-- 139802249115240&#45;&gt;139802249116752 -->
+<g id="edge1" class="edge"><title>139802249115240&#45;&gt;139802249116752</title>
+<path fill="none" stroke="black" d="M74,-1168.31C74,-1160.29 74,-1150.55 74,-1141.57"/>
+<polygon fill="black" stroke="black" points="77.5001,-1141.53 74,-1131.53 70.5001,-1141.53 77.5001,-1141.53"/>
+</g>
+<!-- 139802249116080 -->
+<g id="node3" class="node"><title>139802249116080</title>
+<polygon fill="none" stroke="black" points="45.5,-1022.5 45.5,-1058.5 102.5,-1058.5 102.5,-1022.5 45.5,-1022.5"/>
+<text text-anchor="middle" x="74" y="-1036.8" font-family="Times,serif" font-size="14.00">PReLU</text>
+</g>
+<!-- 139802249116752&#45;&gt;139802249116080 -->
+<g id="edge2" class="edge"><title>139802249116752&#45;&gt;139802249116080</title>
+<path fill="none" stroke="black" d="M74,-1095.31C74,-1087.29 74,-1077.55 74,-1068.57"/>
+<polygon fill="black" stroke="black" points="77.5001,-1068.53 74,-1058.53 70.5001,-1068.53 77.5001,-1068.53"/>
+</g>
+<!-- 139802249141216 -->
+<g id="node4" class="node"><title>139802249141216</title>
+<polygon fill="none" stroke="black" points="43,-949.5 43,-985.5 105,-985.5 105,-949.5 43,-949.5"/>
+<text text-anchor="middle" x="74" y="-963.8" font-family="Times,serif" font-size="14.00">Reshape</text>
+</g>
+<!-- 139802249116080&#45;&gt;139802249141216 -->
+<g id="edge3" class="edge"><title>139802249116080&#45;&gt;139802249141216</title>
+<path fill="none" stroke="black" d="M74,-1022.31C74,-1014.29 74,-1004.55 74,-995.569"/>
+<polygon fill="black" stroke="black" points="77.5001,-995.529 74,-985.529 70.5001,-995.529 77.5001,-995.529"/>
+</g>
+<!-- 139802249139032 -->
+<g id="node5" class="node"><title>139802249139032</title>
+<polygon fill="none" stroke="black" points="11,-876.5 11,-912.5 73,-912.5 73,-876.5 11,-876.5"/>
+<text text-anchor="middle" x="42" y="-890.8" font-family="Times,serif" font-size="14.00">Conv1D</text>
+</g>
+<!-- 139802249141216&#45;&gt;139802249139032 -->
+<g id="edge4" class="edge"><title>139802249141216&#45;&gt;139802249139032</title>
+<path fill="none" stroke="black" d="M66.2537,-949.313C62.5196,-941.028 57.9596,-930.91 53.8055,-921.693"/>
+<polygon fill="black" stroke="black" points="56.9749,-920.207 49.675,-912.529 50.5931,-923.084 56.9749,-920.207"/>
+</g>
+<!-- 139802249138864 -->
+<g id="node8" class="node"><title>139802249138864</title>
+<polygon fill="none" stroke="black" points="46,-657.5 46,-693.5 100,-693.5 100,-657.5 46,-657.5"/>
+<text text-anchor="middle" x="73" y="-671.8" font-family="Times,serif" font-size="14.00">Add</text>
+</g>
+<!-- 139802249141216&#45;&gt;139802249138864 -->
+<g id="edge8" class="edge"><title>139802249141216&#45;&gt;139802249138864</title>
+<path fill="none" stroke="black" d="M77.0668,-949.21C78.8,-938.76 80.8449,-925.154 82,-913 89.695,-832.032 90.4328,-810.895 82,-730 81.1074,-721.437 79.7048,-712.168 78.2702,-703.79"/>
+<polygon fill="black" stroke="black" points="81.6839,-702.997 76.4719,-693.773 74.794,-704.234 81.6839,-702.997"/>
+</g>
+<!-- 139802249141160 -->
+<g id="node6" class="node"><title>139802249141160</title>
+<polygon fill="none" stroke="black" points="2,-803.5 2,-839.5 76,-839.5 76,-803.5 2,-803.5"/>
+<text text-anchor="middle" x="39" y="-817.8" font-family="Times,serif" font-size="14.00">Activation</text>
+</g>
+<!-- 139802249139032&#45;&gt;139802249141160 -->
+<g id="edge5" class="edge"><title>139802249139032&#45;&gt;139802249141160</title>
+<path fill="none" stroke="black" d="M41.2738,-876.313C40.9348,-868.289 40.5231,-858.547 40.1438,-849.569"/>
+<polygon fill="black" stroke="black" points="43.6387,-849.372 39.7195,-839.529 36.6449,-849.668 43.6387,-849.372"/>
+</g>
+<!-- 139802249139312 -->
+<g id="node7" class="node"><title>139802249139312</title>
+<polygon fill="none" stroke="black" points="11,-730.5 11,-766.5 73,-766.5 73,-730.5 11,-730.5"/>
+<text text-anchor="middle" x="42" y="-744.8" font-family="Times,serif" font-size="14.00">Conv1D</text>
+</g>
+<!-- 139802249141160&#45;&gt;139802249139312 -->
+<g id="edge6" class="edge"><title>139802249141160&#45;&gt;139802249139312</title>
+<path fill="none" stroke="black" d="M39.7262,-803.313C40.0652,-795.289 40.4769,-785.547 40.8562,-776.569"/>
+<polygon fill="black" stroke="black" points="44.3551,-776.668 41.2805,-766.529 37.3613,-776.372 44.3551,-776.668"/>
+</g>
+<!-- 139802249139312&#45;&gt;139802249138864 -->
+<g id="edge7" class="edge"><title>139802249139312&#45;&gt;139802249138864</title>
+<path fill="none" stroke="black" d="M49.5042,-730.313C53.0836,-722.115 57.4462,-712.123 61.4363,-702.985"/>
+<polygon fill="black" stroke="black" points="64.771,-704.094 65.5649,-693.529 58.3558,-701.293 64.771,-704.094"/>
+</g>
+<!-- 139802249241544 -->
+<g id="node9" class="node"><title>139802249241544</title>
 <polygon fill="none" stroke="black" points="10,-584.5 10,-620.5 72,-620.5 72,-584.5 10,-584.5"/>
 <text text-anchor="middle" x="41" y="-598.8" font-family="Times,serif" font-size="14.00">Conv1D</text>
 </g>
-<!-- 140387210396840&#45;&gt;140387210276368 -->
-<g id="edge4" class="edge"><title>140387210396840&#45;&gt;140387210276368</title>
+<!-- 139802249138864&#45;&gt;139802249241544 -->
+<g id="edge9" class="edge"><title>139802249138864&#45;&gt;139802249241544</title>
 <path fill="none" stroke="black" d="M65.2537,-657.313C61.5196,-649.028 56.9596,-638.91 52.8055,-629.693"/>
 <polygon fill="black" stroke="black" points="55.9749,-628.207 48.675,-620.529 49.5931,-631.084 55.9749,-628.207"/>
 </g>
-<!-- 140387210936504 -->
-<g id="node8" class="node"><title>140387210936504</title>
+<!-- 139802249241992 -->
+<g id="node12" class="node"><title>139802249241992</title>
 <polygon fill="none" stroke="black" points="45,-365.5 45,-401.5 99,-401.5 99,-365.5 45,-365.5"/>
 <text text-anchor="middle" x="72" y="-379.8" font-family="Times,serif" font-size="14.00">Add</text>
 </g>
-<!-- 140387210396840&#45;&gt;140387210936504 -->
-<g id="edge8" class="edge"><title>140387210396840&#45;&gt;140387210936504</title>
+<!-- 139802249138864&#45;&gt;139802249241992 -->
+<g id="edge13" class="edge"><title>139802249138864&#45;&gt;139802249241992</title>
 <path fill="none" stroke="black" d="M76.0668,-657.21C77.8,-646.76 79.8449,-633.154 81,-621 88.695,-540.032 89.4328,-518.895 81,-438 80.1074,-429.437 78.7048,-420.168 77.2702,-411.79"/>
 <polygon fill="black" stroke="black" points="80.6839,-410.997 75.4719,-401.773 73.794,-412.234 80.6839,-410.997"/>
 </g>
-<!-- 140387210275584 -->
-<g id="node6" class="node"><title>140387210275584</title>
+<!-- 139802249242888 -->
+<g id="node10" class="node"><title>139802249242888</title>
 <polygon fill="none" stroke="black" points="1,-511.5 1,-547.5 75,-547.5 75,-511.5 1,-511.5"/>
 <text text-anchor="middle" x="38" y="-525.8" font-family="Times,serif" font-size="14.00">Activation</text>
 </g>
-<!-- 140387210276368&#45;&gt;140387210275584 -->
-<g id="edge5" class="edge"><title>140387210276368&#45;&gt;140387210275584</title>
+<!-- 139802249241544&#45;&gt;139802249242888 -->
+<g id="edge10" class="edge"><title>139802249241544&#45;&gt;139802249242888</title>
 <path fill="none" stroke="black" d="M40.2738,-584.313C39.9348,-576.289 39.5231,-566.547 39.1438,-557.569"/>
 <polygon fill="black" stroke="black" points="42.6387,-557.372 38.7195,-547.529 35.6449,-557.668 42.6387,-557.372"/>
 </g>
-<!-- 140387210939920 -->
-<g id="node7" class="node"><title>140387210939920</title>
+<!-- 139802249242216 -->
+<g id="node11" class="node"><title>139802249242216</title>
 <polygon fill="none" stroke="black" points="10,-438.5 10,-474.5 72,-474.5 72,-438.5 10,-438.5"/>
 <text text-anchor="middle" x="41" y="-452.8" font-family="Times,serif" font-size="14.00">Conv1D</text>
 </g>
-<!-- 140387210275584&#45;&gt;140387210939920 -->
-<g id="edge6" class="edge"><title>140387210275584&#45;&gt;140387210939920</title>
+<!-- 139802249242888&#45;&gt;139802249242216 -->
+<g id="edge11" class="edge"><title>139802249242888&#45;&gt;139802249242216</title>
 <path fill="none" stroke="black" d="M38.7262,-511.313C39.0652,-503.289 39.4769,-493.547 39.8562,-484.569"/>
 <polygon fill="black" stroke="black" points="43.3551,-484.668 40.2805,-474.529 36.3613,-484.372 43.3551,-484.668"/>
 </g>
-<!-- 140387210939920&#45;&gt;140387210936504 -->
-<g id="edge7" class="edge"><title>140387210939920&#45;&gt;140387210936504</title>
+<!-- 139802249242216&#45;&gt;139802249241992 -->
+<g id="edge12" class="edge"><title>139802249242216&#45;&gt;139802249241992</title>
 <path fill="none" stroke="black" d="M48.5042,-438.313C52.0836,-430.115 56.4462,-420.123 60.4363,-410.985"/>
 <polygon fill="black" stroke="black" points="63.771,-412.094 64.5649,-401.529 57.3558,-409.293 63.771,-412.094"/>
 </g>
-<!-- 140387210289504 -->
-<g id="node9" class="node"><title>140387210289504</title>
+<!-- 139802249240760 -->
+<g id="node13" class="node"><title>139802249240760</title>
 <polygon fill="none" stroke="black" points="9,-292.5 9,-328.5 71,-328.5 71,-292.5 9,-292.5"/>
 <text text-anchor="middle" x="40" y="-306.8" font-family="Times,serif" font-size="14.00">Conv1D</text>
 </g>
-<!-- 140387210936504&#45;&gt;140387210289504 -->
-<g id="edge9" class="edge"><title>140387210936504&#45;&gt;140387210289504</title>
+<!-- 139802249241992&#45;&gt;139802249240760 -->
+<g id="edge14" class="edge"><title>139802249241992&#45;&gt;139802249240760</title>
 <path fill="none" stroke="black" d="M64.2537,-365.313C60.5196,-357.028 55.9596,-346.91 51.8055,-337.693"/>
 <polygon fill="black" stroke="black" points="54.9749,-336.207 47.675,-328.529 48.5931,-339.084 54.9749,-336.207"/>
 </g>
-<!-- 140387210938968 -->
-<g id="node12" class="node"><title>140387210938968</title>
+<!-- 139802249121920 -->
+<g id="node16" class="node"><title>139802249121920</title>
 <polygon fill="none" stroke="black" points="44,-73.5 44,-109.5 98,-109.5 98,-73.5 44,-73.5"/>
 <text text-anchor="middle" x="71" y="-87.8" font-family="Times,serif" font-size="14.00">Add</text>
 </g>
-<!-- 140387210936504&#45;&gt;140387210938968 -->
-<g id="edge13" class="edge"><title>140387210936504&#45;&gt;140387210938968</title>
+<!-- 139802249241992&#45;&gt;139802249121920 -->
+<g id="edge18" class="edge"><title>139802249241992&#45;&gt;139802249121920</title>
 <path fill="none" stroke="black" d="M75.0668,-365.21C76.8,-354.76 78.8449,-341.154 80,-329 87.695,-248.032 88.4328,-226.895 80,-146 79.1074,-137.437 77.7048,-128.168 76.2702,-119.79"/>
 <polygon fill="black" stroke="black" points="79.6839,-118.997 74.4719,-109.773 72.794,-120.234 79.6839,-118.997"/>
 </g>
-<!-- 140387210289840 -->
-<g id="node10" class="node"><title>140387210289840</title>
+<!-- 139802249138528 -->
+<g id="node14" class="node"><title>139802249138528</title>
 <polygon fill="none" stroke="black" points="0,-219.5 0,-255.5 74,-255.5 74,-219.5 0,-219.5"/>
 <text text-anchor="middle" x="37" y="-233.8" font-family="Times,serif" font-size="14.00">Activation</text>
 </g>
-<!-- 140387210289504&#45;&gt;140387210289840 -->
-<g id="edge10" class="edge"><title>140387210289504&#45;&gt;140387210289840</title>
+<!-- 139802249240760&#45;&gt;139802249138528 -->
+<g id="edge15" class="edge"><title>139802249240760&#45;&gt;139802249138528</title>
 <path fill="none" stroke="black" d="M39.2738,-292.313C38.9348,-284.289 38.5231,-274.547 38.1438,-265.569"/>
 <polygon fill="black" stroke="black" points="41.6387,-265.372 37.7195,-255.529 34.6449,-265.668 41.6387,-265.372"/>
 </g>
-<!-- 140384853547776 -->
-<g id="node11" class="node"><title>140384853547776</title>
+<!-- 139802249122032 -->
+<g id="node15" class="node"><title>139802249122032</title>
 <polygon fill="none" stroke="black" points="9,-146.5 9,-182.5 71,-182.5 71,-146.5 9,-146.5"/>
 <text text-anchor="middle" x="40" y="-160.8" font-family="Times,serif" font-size="14.00">Conv1D</text>
 </g>
-<!-- 140387210289840&#45;&gt;140384853547776 -->
-<g id="edge11" class="edge"><title>140387210289840&#45;&gt;140384853547776</title>
+<!-- 139802249138528&#45;&gt;139802249122032 -->
+<g id="edge16" class="edge"><title>139802249138528&#45;&gt;139802249122032</title>
 <path fill="none" stroke="black" d="M37.7262,-219.313C38.0652,-211.289 38.4769,-201.547 38.8562,-192.569"/>
 <polygon fill="black" stroke="black" points="42.3551,-192.668 39.2805,-182.529 35.3613,-192.372 42.3551,-192.668"/>
 </g>
-<!-- 140384853547776&#45;&gt;140387210938968 -->
-<g id="edge12" class="edge"><title>140384853547776&#45;&gt;140387210938968</title>
+<!-- 139802249122032&#45;&gt;139802249121920 -->
+<g id="edge17" class="edge"><title>139802249122032&#45;&gt;139802249121920</title>
 <path fill="none" stroke="black" d="M47.5042,-146.313C51.0836,-138.115 55.4462,-128.123 59.4363,-118.985"/>
 <polygon fill="black" stroke="black" points="62.771,-120.094 63.5649,-109.529 56.3558,-117.293 62.771,-120.094"/>
 </g>
-<!-- 140387210275864 -->
-<g id="node13" class="node"><title>140387210275864</title>
+<!-- 139802249140656 -->
+<g id="node17" class="node"><title>139802249140656</title>
 <polygon fill="none" stroke="black" points="44,-0.5 44,-36.5 98,-36.5 98,-0.5 44,-0.5"/>
 <text text-anchor="middle" x="71" y="-14.8" font-family="Times,serif" font-size="14.00">Flatten</text>
 </g>
-<!-- 140387210938968&#45;&gt;140387210275864 -->
-<g id="edge14" class="edge"><title>140387210938968&#45;&gt;140387210275864</title>
+<!-- 139802249121920&#45;&gt;139802249140656 -->
+<g id="edge19" class="edge"><title>139802249121920&#45;&gt;139802249140656</title>
 <path fill="none" stroke="black" d="M71,-73.3129C71,-65.2895 71,-55.5475 71,-46.5691"/>
 <polygon fill="black" stroke="black" points="74.5001,-46.5288 71,-36.5288 67.5001,-46.5289 74.5001,-46.5288"/>
 </g>

scipydiffe.ipynb

@@ -0,0 +1,649 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 413,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-11T10:18:28.370039Z",
+     "start_time": "2018-10-11T10:18:28.148656Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n",
+      "Dataset has been loaded\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import h5py\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "#import jtplot submodule from jupyterthemes\n",
+    "from jupyterthemes import jtplot\n",
+    "#currently installed theme will be used to\n",
+    "#set plot style if no arguments provided\n",
+    "jtplot.style()\n",
+    "\n",
+    "#now load this dataset \n",
+    "h5f = h5py.File('./datasets/s8_sio2tio2_v2.h5','r')\n",
+    "X = h5f['sizes'][:]\n",
+    "Y = h5f['spectrum'][:]\n",
+    "\n",
+    "#create a train - test split of the dataset\n",
+    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.4, random_state=42)\n",
+    "# normalize inputs \n",
+    "#x_test = (x_test - 50)/20 \n",
+    "\n",
+    "print(\"Dataset has been loaded\")\n",
+    "\n",
+    "import numpy as np\n",
+    "import mxnet as mx\n",
+    "\n",
+    "# Step1: Load the model in MXNet\n",
+    "\n",
+    "# Use the same prefix and epoch parameters we used in save_mxnet_model API.\n",
+    "sym, arg_params, aux_params = mx.model.load_checkpoint(prefix='my_mod_fullycon', epoch=0)\n",
+    "\n",
+    "# We use the data_names and data_shapes returned by save_mxnet_model API.\n",
+    "mod = mx.mod.Module(symbol=sym, \n",
+    "                    data_names=['/first_input2'], \n",
+    "                    context=mx.gpu(), \n",
+    "                    label_names=None)\n",
+    "mod.bind(for_training=False, \n",
+    "         data_shapes=[('/first_input2', (1,8))], \n",
+    "         label_shapes=mod._label_shapes)\n",
+    "mod.set_params(arg_params, aux_params, allow_missing=True)    \n",
+    "\n",
+    "\n",
+    "#resnet - my_mod_bet   input_21\n",
+    "#fullycon - my_mod_fullycon  first_input2\n",
+    "#conv1d - my_mod_conv1d - first_input4\n",
+    "#convprel - my_mod_convprel - first_input6"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 341,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-10T06:51:51.680142Z",
+     "start_time": "2018-10-10T06:51:51.642525Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "def de_stage2(fobj, bounds, popint, history, itprev, mut=0.8, crossp=0.7, popsize=20, its=1000):\n",
+    "    #history=[]\n",
+    "    dimensions = len(bounds)\n",
+    "    min_b, max_b = np.asarray(bounds).T\n",
+    "    diff = np.fabs(min_b - max_b)\n",
+    "    pop_denorm = min_b + pop * diff\n",
+    "    fitness = np.asarray([fobj(ind) for ind in pop_denorm])\n",
+    "    best_idx = np.argmin(fitness)\n",
+    "    best = pop_denorm[best_idx]\n",
+    "    for i in range(its):\n",
+    "        #trialarr = np.zeros(popsize)\n",
+    "        for j in range(popsize):\n",
+    "            idxs = [idx for idx in range(popsize) if idx != j]\n",
+    "            a, b, c = pop[np.random.choice(idxs, 3, replace = False)]\n",
+    "            #mutant = np.clip(a + mut * (b - c), 0, 1)\n",
+    "            mutant = np.clip(pop[best_idx] + mut * (b - c), 0, 1)\n",
+    "            cross_points = np.random.rand(dimensions) < crossp\n",
+    "            if not np.any(cross_points):\n",
+    "                cross_points[np.random.randint(0, dimensions)] = True\n",
+    "            trial = np.where(cross_points, mutant, pop[j])\n",
+    "            #trialarr[j] = np.where(cross_points, mutant, pop[j])\n",
+    "            trial_denorm = min_b + trial * diff\n",
+    "            f = fobj(trial_denorm)\n",
+    "            if f < fitness[j]:\n",
+    "                fitness[j] = f\n",
+    "                pop[j] = trial\n",
+    "                if f < fitness[best_idx]:\n",
+    "                    best_idx = j\n",
+    "                    best = trial_denorm\n",
+    "        if i%20 == 0:\n",
+    "            print(i, fitness[best_idx])\n",
+    "        history.append([i+itprev, fitness[best_idx]])\n",
+    "    return best, fitness[best_idx], history\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "def de(fobj, bounds, mut=0.8, crossp=0.7, popsize=20, its=1000):\n",
+    "    dimensions = len(bounds)\n",
+    "    history=[]\n",
+    "    pop = np.random.rand(popsize, dimensions)\n",
+    "    min_b, max_b = np.asarray(bounds).T\n",
+    "    diff = np.fabs(min_b - max_b)\n",
+    "    pop_denorm = min_b + pop * diff\n",
+    "    fitness = np.asarray([fobj(ind) for ind in pop_denorm])\n",
+    "    best_idx = np.argmin(fitness)\n",
+    "    best = pop_denorm[best_idx]\n",
+    "    for i in range(its):\n",
+    "        #trialarr = np.zeros(popsize)\n",
+    "        for j in range(popsize):\n",
+    "            idxs = [idx for idx in range(popsize) if idx != j]\n",
+    "            a, b, c = pop[np.random.choice(idxs, 3, replace = False)]\n",
+    "            #mutant = np.clip(a + mut * (b - c), 0, 1)\n",
+    "            mutant = np.clip(pop[best_idx] + mut * (b - c), 0, 1)\n",
+    "            cross_points = np.random.rand(dimensions) < crossp\n",
+    "            if not np.any(cross_points):\n",
+    "                cross_points[np.random.randint(0, dimensions)] = True\n",
+    "            trial = np.where(cross_points, mutant, pop[j])\n",
+    "            #trialarr[j] = np.where(cross_points, mutant, pop[j])\n",
+    "            trial_denorm = min_b + trial * diff\n",
+    "            f = fobj(trial_denorm)\n",
+    "            if f < fitness[j]:\n",
+    "                fitness[j] = f\n",
+    "                pop[j] = trial\n",
+    "                if f < fitness[best_idx]:\n",
+    "                    best_idx = j\n",
+    "                    best = trial_denorm\n",
+    "        if i%25 == 0:\n",
+    "            print(i, fitness[best_idx])\n",
+    "        history.append([i, fitness[best_idx]])\n",
+    "    return best, fitness[best_idx], history\n",
+    "\n",
+    "\n",
+    "def de2(fobj, bounds, mut=0.8, crossp=0.7, popsize=20, its=1000):\n",
+    "    dimensions = len(bounds)\n",
+    "    history=[]\n",
+    "    pop = np.random.rand(popsize, dimensions)\n",
+    "    min_b, max_b = np.asarray(bounds).T\n",
+    "    diff = np.fabs(min_b - max_b)\n",
+    "    pop_denorm = min_b + pop * diff\n",
+    "    fitness = np.asarray(fobj(pop_denorm))\n",
+    "    best_idx = np.argmin(fitness)\n",
+    "    best = pop_denorm[best_idx]\n",
+    "    for i in range(its):\n",
+    "        trialarr = np.zeros((popsize, dimensions))\n",
+    "        for j in range(popsize):\n",
+    "            idxs = [idx for idx in range(popsize) if idx != j]\n",
+    "            a, b, c = pop[np.random.choice(idxs, 3, replace = False)]\n",
+    "            #mutant = np.clip(a + mut * (b - c), 0, 1)\n",
+    "            mutant = np.clip(pop[best_idx] + mut * (b - c), 0, 1)\n",
+    "            cross_points = np.random.rand(dimensions) < crossp\n",
+    "            if not np.any(cross_points):\n",
+    "                cross_points[np.random.randint(0, dimensions)] = True\n",
+    "            trialarr[j] = np.where(cross_points, mutant, pop[j])\n",
+    "            \n",
+    "        \n",
+    "        trial_denorm = min_b + trialarr * diff\n",
+    "        #print(trial_denorm)\n",
+    "        f = fobj(trial_denorm)\n",
+    "        for j in range(popsize):\n",
+    "            if f[j] < fitness[j]:\n",
+    "                fitness[j] = f[j]\n",
+    "                pop[j] = trialarr[j]\n",
+    "                if f[j] < fitness[best_idx]:\n",
+    "                    best_idx = j\n",
+    "                    best = trial_denorm[j]\n",
+    "        if i%50 == 0:\n",
+    "            print(i, fitness[best_idx])\n",
+    "        history.append([i, fitness[best_idx]])\n",
+    "    return pop, fitness, best, history\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 414,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-11T10:32:13.136288Z",
+     "start_time": "2018-10-11T10:18:36.758693Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "iteration  0\n",
+      "0 1.1705265\n",
+      "50 0.4470625\n",
+      "100 0.32243434\n",
+      "150 0.28111637\n",
+      "200 0.21437946\n",
+      "250 0.20201741\n",
+      "300 0.16932511\n",
+      "350 0.16932511\n",
+      "400 0.16932511\n",
+      "450 0.16185972\n",
+      "0 0.3121147107485763\n",
+      "20 0.3121147107485763\n",
+      "40 0.20575947392573551\n",
+      "60 0.20575947392573551\n",
+      "80 0.20575947392573551\n",
+      "100 0.20575947392573551\n",
+      "120 0.1619773697186888\n",
+      "140 0.1619773697186888\n",
+      "160 0.1619773697186888\n",
+      "180 0.14528643576910827\n",
+      "200 0.1421019546761191\n",
+      "220 0.12192860901063259\n",
+      "240 0.09253242337067973\n",
+      "260 0.08082859150315144\n",
+      "280 0.07013973087543791\n",
+      "iteration  1\n",
+      "0 1.2500818\n",
+      "50 0.3227382\n",
+      "100 0.22907153\n",
+      "150 0.21646158\n",
+      "200 0.20283288\n",
+      "250 0.1756429\n",
+      "300 0.16450137\n",
+      "350 0.15796128\n",
+      "400 0.15796128\n",
+      "450 0.1558615\n",
+      "0 0.24526450596507823\n",
+      "20 0.24526450596507823\n",
+      "40 0.24526450596507823\n",
+      "60 0.24526450596507823\n",
+      "80 0.24526450596507823\n",
+      "100 0.24526450596507823\n",
+      "120 0.24526450596507823\n",
+      "140 0.22005031308025427\n",
+      "160 0.12697402424132934\n",
+      "180 0.12697402424132934\n",
+      "200 0.12697402424132934\n",
+      "220 0.12697402424132934\n",
+      "240 0.11092008062789101\n",
+      "260 0.07282972818333444\n",
+      "280 0.07208495130811798\n",
+      "iteration  2\n",
+      "0 1.1091585\n",
+      "50 0.37814587\n",
+      "100 0.3192608\n",
+      "150 0.2471918\n",
+      "200 0.19585466\n",
+      "250 0.19585466\n",
+      "300 0.18117855\n",
+      "350 0.17847842\n",
+      "400 0.1716342\n",
+      "450 0.16229132\n",
+      "0 0.37607692325889464\n",
+      "20 0.37607692325889464\n",
+      "40 0.37607692325889464\n",
+      "60 0.3330564707656615\n",
+      "80 0.3330564707656615\n",
+      "100 0.3330564707656615\n",
+      "120 0.3054040753273184\n",
+      "140 0.30056595386813395\n",
+      "160 0.2372560274974856\n",
+      "180 0.2342672435510109\n",
+      "200 0.22194270193570056\n",
+      "220 0.20433060781452445\n",
+      "240 0.2008148449307781\n",
+      "260 0.19259466427441313\n",
+      "280 0.19259466427441313\n",
+      "iteration  3\n",
+      "0 1.896904\n",
+      "50 0.3967435\n",
+      "100 0.2679344\n",
+      "150 0.17716393\n",
+      "200 0.17716393\n",
+      "250 0.17716393\n",
+      "300 0.16917363\n",
+      "350 0.16231227\n",
+      "400 0.16231227\n",
+      "450 0.1614473\n",
+      "0 0.3608929001962287\n",
+      "20 0.3608929001962287\n",
+      "40 0.3608929001962287\n",
+      "60 0.3608929001962287\n",
+      "80 0.3608929001962287\n",
+      "100 0.2580806115772493\n",
+      "120 0.23025322555288447\n",
+      "140 0.181724619284048\n",
+      "160 0.15870143348935536\n",
+      "180 0.15870143348935536\n",
+      "200 0.1406708686897299\n",
+      "220 0.11690700185881761\n",
+      "240 0.11690700185881761\n",
+      "260 0.11548268043464019\n",
+      "280 0.10501889934993124\n",
+      "iteration  4\n",
+      "0 2.6791306\n",
+      "50 0.37418172\n",
+      "100 0.32684445\n",
+      "150 0.22900505\n",
+      "200 0.22496085\n",
+      "250 0.21092188\n",
+      "300 0.20759842\n",
+      "350 0.17270699\n",
+      "400 0.16482411\n",
+      "450 0.16482411\n",
+      "0 0.6541415220770362\n",
+      "20 0.6541415220770362\n",
+      "40 0.6541415220770362\n",
+      "60 0.6541415220770362\n",
+      "80 0.6541415220770362\n",
+      "100 0.6022119287555459\n",
+      "120 0.5550346370303875\n",
+      "140 0.5550346370303875\n",
+      "160 0.5550346370303875\n",
+      "180 0.5129769561955577\n",
+      "200 0.5129769561955577\n",
+      "220 0.5073713851357137\n",
+      "240 0.4369662361659122\n",
+      "260 0.4369662361659122\n",
+      "280 0.4369662361659122\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import numpy as np\n",
+    "from scipy.optimize import rosen, differential_evolution\n",
+    "import scnets as scn\n",
+    "import mxnet as mx\n",
+    "from mxnet import nd\n",
+    "import snlay as snlay\n",
+    "import time\n",
+    "\n",
+    "\n",
+    "mats = np.array([3, 4, 3, 4, 3, 4, 3, 4])\n",
+    "lams = np.linspace(300, 1200, 256)\n",
+    "targ_spec = y_test[29]\n",
+    "targ_spec2 = np.tile(targ_spec, (psize,1))\n",
+    "\n",
+    "# def loss_jumper(x, it_cnt=0):\n",
+    "\n",
+    "def mxmod_arr_loss(x):\n",
+    "    x_np = np.array(x)\n",
+    "    x_np = (x_np - 50.0)/20.0\n",
+    "    res2 = mod.predict(x_np)\n",
+    "    y_t = nd.array(targ_spec2, ctx=mx.gpu())\n",
+    "    err = nd.abs(y_t - res2)/y_t\n",
+    "    err2 = 100*nd.mean(err, axis=1).asnumpy()\n",
+    "    return err2\n",
+    "\n",
+    "def mxmod_loss(x):\n",
+    "    x_np = np.array(x)\n",
+    "    x_np = (x_np - 50.0)/20.0\n",
+    "    x_np = np.expand_dims(x_np, axis=0)\n",
+    "    res2 = mod.predict(x_np)\n",
+    "    y_t = nd.array(targ_spec, ctx=mx.gpu())\n",
+    "    err = nd.abs(y_t - res2)/y_t\n",
+    "    err2 = 100*nd.mean(err).asscalar()\n",
+    "    return err2\n",
+    "\n",
+    "def loss_func(x):\n",
+    "    #x_np = np.array(x)\n",
+    "    #count+=1\n",
+    "    spec_ac = snlay.calc_spectrum(x, mats, lams)\n",
+    "    diff = np.abs(targ_spec - spec_ac)/targ_spec\n",
+    "    return 100*np.amax(diff)\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "bnds = [(30, 70)]*8\n",
+    "\n",
+    "psize = 640\n",
+    "its_first = 500\n",
+    "psnew = 20\n",
+    "its_second = 300\n",
+    "reps = 5\n",
+    "run_hist = []\n",
+    "run_time1 = []\n",
+    "run_time2 = []\n",
+    "run_time_tot = []\n",
+    "run_pmre = []\n",
+    "run_best = []\n",
+    "\n",
+    "for rep in range(reps):\n",
+    "    print(\"iteration \", rep)\n",
+    "    start = time.time()\n",
+    "    pop, f, b, hstry = de2(fobj=mxmod_arr_loss, bounds=bnds, popsize=psize, its=its_first) \n",
+    "    end1 = time.time()\n",
+    "    marg = int(psnew/5)\n",
+    "    pnew1 = pop[np.argsort(f)][:psnew-marg]\n",
+    "    pnew2 = pop[np.argsort(f)][psnew-marg:psnew]\n",
+    "    pnew = np.concatenate((pnew1, pnew2))\n",
+    "    b, c, hstry = de_stage2(fobj=loss_func, bounds=bnds, popint=pnew, history=hstry, itprev=its_first, popsize=psnew, its=its_second)\n",
+    "    end = time.time()\n",
+    "    run_time1.append((end1 - start)/60.0)\n",
+    "    run_time2.append((end - end1)/60.0)\n",
+    "    run_time_tot.append((end - start)/60.0)\n",
+    "    run_pmre.append(c)\n",
+    "    run_best.append(b)\n",
+    "    run_hist.append(np.asarray(hstry))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 415,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-11T10:54:05.648366Z",
+     "start_time": "2018-10-11T10:54:05.621923Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0.06614759 0.07013973 0.09779269 0.19104877 0.43696624]\n",
+      "[2.70871968 2.71356018 2.71965878 2.72612135 2.73761866]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(np.sort(run_pmre))\n",
+    "print(np.sort(run_time_tot))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 303,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-10T06:20:12.311685Z",
+     "start_time": "2018-10-10T06:20:12.284836Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([30., 64., 51., 37., 37., 64., 35., 62.])"
+      ]
+     },
+     "execution_count": 303,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.round(b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 304,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-10T06:20:17.206532Z",
+     "start_time": "2018-10-10T06:20:17.183258Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([30., 64., 52., 36., 36., 64., 35., 62.])"
+      ]
+     },
+     "execution_count": 304,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x_test[29]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 284,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T18:25:18.944088Z",
+     "start_time": "2018-10-05T18:09:14.380943Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0 34.764084177464106\n",
+      "25 8.340754312117314\n",
+      "50 5.596934152569711\n",
+      "75 2.4355046393850808\n",
+      "100 2.4355046393850808\n",
+      "125 2.4134250007110736\n",
+      "150 1.7133162654256295\n",
+      "175 1.7133162654256295\n",
+      "200 1.657703207258774\n",
+      "225 1.357276014324554\n",
+      "250 1.2038289909480697\n",
+      "275 1.2038289909480697\n",
+      "300 1.0010963661625447\n",
+      "325 0.9091072669949225\n",
+      "350 0.8380571414952189\n",
+      "375 0.6046218491740213\n",
+      "400 0.6046218491740213\n",
+      "425 0.5006209308298584\n",
+      "450 0.45559701169843764\n",
+      "475 0.4376195672622949\n",
+      "500 0.3938599657695569\n",
+      "525 0.25238263922875026\n",
+      "550 0.16105660067589186\n",
+      "575 0.16105660067589186\n"
+     ]
+    }
+   ],
+   "source": [
+    "b, c = de(fobj=loss_func, bounds=bnds, popsize=80, its=600)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 286,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-05T18:27:24.231716Z",
+     "start_time": "2018-10-05T18:27:24.206383Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([30., 64., 52., 36., 36., 64., 35., 62.])"
+      ]
+     },
+     "execution_count": 286,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.round(b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 418,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-10-13T05:37:08.770741Z",
+     "start_time": "2018-10-13T05:37:08.742574Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "7.984925229121"
+      ]
+     },
+     "execution_count": 418,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "41**8/(1e12)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.4"
+  },
+  "latex_envs": {
+   "LaTeX_envs_menu_present": true,
+   "autoclose": false,
+   "autocomplete": true,
+   "bibliofile": "biblio.bib",
+   "cite_by": "apalike",
+   "current_citInitial": 1,
+   "eqLabelWithNumbers": true,
+   "eqNumInitial": 1,
+   "hotkeys": {
+    "equation": "Ctrl-E",
+    "itemize": "Ctrl-I"
+   },
+   "labels_anchors": false,
+   "latex_user_defs": false,
+   "report_style_numbering": false,
+   "user_envs_cfg": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}