nengo_3layer_nodenselayer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import gzip\n",
    "import pickle\n",
    "from urllib.request import urlretrieve\n",
    "import zipfile\n",
    "\n",
    "import nengo\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import nengo_dl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAB5ZJREFUeJzt3T+I1/cdx/Hvp7GhxJocLgpCCDqcmBBuiUKQ2iAigQi9xKHSTAmZcuDkki1QrZA/g9XhpkAWcTRJoToYFRoQJOpy1KFbwi0lOY0mKuo3S+hSf+8zvzvvd97r8QAXX3z9fgeffNXP/c7W930HrHy/GfUDAEtD7BBC7BBC7BBC7BBC7BBC7BBC7DxQa+1sa+1Wa+3GLz+ujvqZWBixU5nq+/73v/wYH/XDsDBihxBip/K31tp/W2v/aq39cdQPw8I0XxvPg7TWtnVdN9N13Z2u6/7cdd3Rrusm+r7/z0gfjKGJnYfSWvtn13X/6Pv+76N+Fobjj/E8rL7rujbqh2B4Yuf/tNbGWmu7W2u/a62taq39peu6P3Rdd2rUz8bwVo36AViWftt13V+7rtvcdd29ruv+3XXdn/q+d9b+GPN3dgjhj/EQQuwQQuwQQuwQYkn/Nb615l8D4RHr+/6BXw/hzQ4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hxA4hVo36AXi0nnjiiXJ/5plnHun9p6amBm5PPfVUee34+Hi5v/vuu+X+4YcfDtz27dtXXnvr1q1yP3z4cLm///775T4K3uwQQuwQQuwQQuwQQuwQQuwQQuwQwjn7Enj22WfL/cknnyz3l19+udy3b98+cBsbGyuvfeONN8p9lL755ptyP3LkSLlPTk4O3H744Yfy2itXrpT7uXPnyn058maHEGKHEGKHEGKHEGKHEGKHEK3v+6W7WWtLd7MlNDExUe5nzpwp90f9MdPl6v79++X+1ltvlfuNGzeGvvfs7Gy5f//99+V+9erVoe/9qPV93x70897sEELsEELsEELsEELsEELsEELsEMI5+yJYu3ZtuV+4cKHcN27cuJiPs6jme/a5ublyf+WVVwZud+7cKa9N/fqDhXLODuHEDiHEDiHEDiHEDiHEDiHEDiF8K+lF8N1335X7gQMHyv21114r90uXLpX7fN9SuXL58uVy37VrV7nfvHmz3J9//vmB2/79+8trWVze7BBC7BBC7BBC7BBC7BBC7BBC7BDC59mXgaeffrrc5/vvhaenpwdub7/9dnntm2++We7Hjx8vd5Yfn2eHcGKHEGKHEGKHEGKHEGKHEGKHED7Pvgxcv359Qddfu3Zt6Gvfeeedcj9x4kS5z/d/rLN8eLNDCLFDCLFDCLFDCLFDCLFDCB9xXQFWr149cPv888/La3fs2FHur776armfPn263Fl6PuIK4cQOIcQOIcQOIcQOIcQOIcQOIZyzr3CbNm0q96+//rrc5+bmyv3LL78s94sXLw7cjh07Vl67lL83VxLn7BBO7BBC7BBC7BBC7BBC7BBC7BDCOXu4ycnJcv/kk0/Kfc2aNUPf+7333iv3Tz/9tNxnZ2eHvvdK5pwdwokdQogdQogdQogdQogdQogdQjhnp/TCCy+U+8cff1zuO3fuHPre09PT5X7w4MFy//bbb4e+9+PMOTuEEzuEEDuEEDuEEDuEEDuEEDuEcM7OgoyNjZX7nj17Bm7zfVa+tQceF//PmTNnyn3Xrl3lvlI5Z4dwYocQYocQYocQYocQYocQjt4Ymdu3b5f7qlWryv3u3bvlvnv37oHb2bNny2sfZ47eIJzYIYTYIYTYIYTYIYTYIYTYIUR9kEm8F198sdz37t1b7i+99NLAbb5z9PnMzMyU+/nz5xf066803uwQQuwQQuwQQuwQQuwQQuwQQuwQwjn7Cjc+Pl7uU1NT5f7666+X+/r163/1Mz2se/fulfvs7Gy5379/fzEf57HnzQ4hxA4hxA4hxA4hxA4hxA4hxA4hnLM/BuY7y963b9/Abb5z9Oeee26YR1oUFy9eLPeDBw+W+2effbaYj7PiebNDCLFDCLFDCLFDCLFDCLFDCEdvS2DdunXlvmXLlnI/evRouW/evPlXP9NiuXDhQrl/8MEHA7eTJ0+W1/qI6uLyZocQYocQYocQYocQYocQYocQYocQztkf0tq1awdu09PT5bUTExPlvnHjxqGeaTF89dVX5f7RRx+V+6lTp8r9p59++tXPxKPhzQ4hxA4hxA4hxA4hxA4hxA4hxA4hYs7Zt23bVu4HDhwo961btw7cNmzYMNQzLZYff/xx4HbkyJHy2kOHDpX7zZs3h3omlh9vdgghdgghdgghdgghdgghdgghdggRc84+OTm5oH0hZmZmyv2LL74o97t375Z79Znzubm58lpyeLNDCLFDCLFDCLFDCLFDCLFDCLFDiNb3/dLdrLWluxmE6vu+PejnvdkhhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghxJJ+K2lgdLzZIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIcTPz+xkYgSG6RgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAB/9JREFUeJzt3UGInPUdxvH/GyUhNt2F5tAIwVtyaUlWgxJ6MXhUQcGDpOKeCtIgSJBAhU3ITRAVVBoRRNAmdEWkqKTiZcWLJ0n00It4qVgDiRLibgMJxLeHWqmy85u4O87s7vP5HOfhzbwx+fLKvnlnur7vG7DxbZr0CQDjIXYIIXYIIXYIIXYIIXYIIXYIIXaW1XXdr7qu+1vXdf/uuu6fXdf9ftLnxOrcOOkTYM36c2vtamvt1621mdba6a7rPun7/h+TPS1WqvMv6Pixrut+0Vq72Fr7bd/3n3732l9aa//q+/5PEz05Vsz/xrOc3a21a/8L/TuftNZ+M6HzYQTEznK2tdYu/ei1S621X07gXBgRsbOcpdba1I9em2qtLU7gXBgRsbOcT1trN3Zdt+v/XtvbWvPDuXXMD+hYVtd18621vrX2h/bfn8b/vbX2Oz+NX79c2RnkUGtta2vtfGvtr621Pwp9fXNlhxCu7BBC7BBC7BBC7BBirA/CdF3np4HwM+v7vlvudVd2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CDHWr2xm49m3b1+5P/roowO32dnZ8tjXXnut3F944YVyP3PmTLmncWWHEGKHEGKHEGKHEGKHEGKHEGKHEF3f9+N7s64b35sxEjMzM+W+sLBQ7lNTU6M8nR+4dOlSuW/fvv1ne++1rO/7brnXXdkhhNghhNghhNghhNghhNghhNghhOfZw91xxx3l/uabb5b79PR0uVf/jmNxcbE89urVq+U+7D76/v37B27DnnUf9t7rkSs7hBA7hBA7hBA7hBA7hBA7hPCI6wZw0003Ddxuu+228tiTJ0+W+86dO8u965Z9mvJ71d+vYbe/nnrqqXKfn58v9+rc5ubmymOffPLJcl/LPOIK4cQOIcQOIcQOIcQOIcQOIcQOITziugG89NJLA7eDBw+O8Ux+mmH/BmDbtm3l/sEHH5T7gQMHBm579uwpj92IXNkhhNghhNghhNghhNghhNghhNghhPvs68C+ffvK/Z577hm4DXvefJhh97Lfeeedcn/66acHbl9++WV57NmzZ8v94sWL5X7XXXcN3Fb732U9cmWHEGKHEGKHEGKHEGKHEGKHEGKHED43fg2YmZkp94WFhXKfmppa8Xu/++675T7sefg777yz3Kvnxl9++eXy2AsXLpT7MNeuXRu4Xb58uTx22O9r2GfeT5LPjYdwYocQYocQYocQYocQYocQYocQnmcfg927d5f7kSNHyn16errcv/rqq4HbuXPnymNfffXVcl9aWir306dPr2qflK1bt5b7448/Xu4PPfTQKE9nLFzZIYTYIYTYIYTYIYTYIYTYIYRbbyOwZcuWcq8+Trm11u6+++5yX1xcLPfZ2dmB20cffVQeO+wWVKpbbrll0qcwcq7sEELsEELsEELsEELsEELsEELsEMJ99hG49dZby33YffRh7rvvvnIf9rXK0JorO8QQO4QQO4QQO4QQO4QQO4QQO4Rwn30Enn322XLvumW/Qfd7w+6Tu4++Mps2Db6Wffvtt2M8k7XBlR1CiB1CiB1CiB1CiB1CiB1CiB1CuM9+ne69996B28zMTHls3/fl/vbbb6/onKhV99KH/Zl8/PHHoz6diXNlhxBihxBihxBihxBihxBihxBihxDus1+n6nvMN2/eXB57/vz5cn/99ddXdE4b3bDvvT9+/PiKf+2FhYVyf+KJJ1b8a69VruwQQuwQQuwQQuwQQuwQQuwQwq23Mbhy5Uq5nzt3bkxnsrYMu7U2NzdX7keOHCn3L774YuD2zDPPlMcuLS2V+3rkyg4hxA4hxA4hxA4hxA4hxA4hxA4h3Gcfg+SPiq4+ZnvYffIHH3yw3N96661yf+CBB8o9jSs7hBA7hBA7hBA7hBA7hBA7hBA7hHCf/Tp1XbeirbXW7r///nJ/7LHHVnROa8Hhw4fL/ejRowO36enp8thTp06V++zsbLnzQ67sEELsEELsEELsEELsEELsEELsEMJ99uvU9/2KttZa27FjR7k///zz5f7KK6+U+9dffz1w279/f3nsww8/XO579+4t9507d5b7559/PnB77733ymNPnDhR7vw0ruwQQuwQQuwQQuwQQuwQQuwQwq23MbjhhhvK/dChQ+U+7CORv/nmm4Hbrl27ymNX68MPPyz3999/f+B27NixUZ8OBVd2CCF2CCF2CCF2CCF2CCF2CCF2CNENezxzpG/WdeN7sxGrHuV84403ymNvv/32Vb33sI+qXs2fYfV4bGutzc/Pl/t6/hjsjarv+2X/wriyQwixQwixQwixQwixQwixQwixQwj32Ufg5ptvLvdHHnmk3Ofm5sp9NffZn3vuufLYF198sdw/++yzcmftcZ8dwokdQogdQogdQogdQogdQogdQrjPDhuM++wQTuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQQuwQYqxf2QxMjis7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hBA7hPgPUQBlTrx8ueoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABphJREFUeJzt3b9r1fsdx/HPJ1cES4moiAiC10GFDrEtOOigQkTEXSgobip1bf+COgmudbLgoKPEzUHu4KQSHf0NglYH6SJY4hWEb4f2wu1tzufU5Hi+Oef1eGzmzdfzJuTJB/LJSWrXdQWYfjN9LwCMh9ghhNghhNghhNghhNghhNghhNgZqNa6u9b6Y631et+7sHpip+WvpZTFvpdgNMTOsmqtfyilfCil/ND3LoyG2PkftdbZUspfSil/6nsXRkfsLOdiKeVvXdf9ve9FGJ11fS/A2lJr/W0p5Wgp5Xd978JoiZ1fOlJK+b6U8qbWWkopvy6lfFdr/U3Xdb/vcS9WqXqLKz9Xa/1VKWX2Zx/6c/l3/H/suu4fvSzFSDjZ+S9d1y2VUpZ++net9Z+llB+FPvmc7BDCd+MhhNghhNghhNghxFi/G19r9d1A+Ma6rqvLfdzJDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHEDiHW9b0AfCvz8/MDZzdu3Gg+e/jw4eb8+fPnK9qpT052CCF2CCF2CCF2CCF2CCF2CCF2CBFzz37o0KHmfMuWLc35wsLCKNdhDPbv3z9wtri4OMZN1gYnO4QQO4QQO4QQO4QQO4QQO4SIuXo7cuRIc7579+7m3NXb2jMz0z6rdu3aNXC2c+fO5rO11hXttJY52SGE2CGE2CGE2CGE2CGE2CGE2CFEzD37mTNnmvN79+6NaRNGZfv27c352bNnB86uX7/efPbZs2cr2mktc7JDCLFDCLFDCLFDCLFDCLFDCLFDiJh79mHvfWbyXL16dcXPvnz5coSbTAYFQAixQwixQwixQwixQwixQwixQ4ipuWefm5trzrdt2zamTRiXjRs3rvjZO3fujHCTyeBkhxBihxBihxBihxBihxBihxBihxBTc89+4sSJ5nzDhg1j2oRRGfazEa2/vz7Mu3fvVvzspHKyQwixQwixQwixQwixQwixQ4ipuXrbu3fvqp5//PjxiDZhVC5fvtycD7uae/HixcDZx48fV7TTJHOyQwixQwixQwixQwixQwixQwixQ4ipuWdfrcXFxb5XmEizs7PN+fHjxwfOTp8+3Xz22LFjK9rpJxcvXhw4+/Dhw6r+70nkZIcQYocQYocQYocQYocQYocQYocQ7tn/Y/Pmzb299r59+5rzWmtzfvTo0YGzHTt2NJ9dv359c37q1KnmfGamfV58+vRp4OzBgwfNZz9//tycr1vX/vJ99OhRc57GyQ4hxA4hxA4hxA4hxA4hxA4hxA4hatd143uxWr/Zi125cqU5P3/+fHM+7P3Nb968+eqd/l9zc3PN+bB79i9fvgycLS0tNZ998uRJcz7sLvzhw4fN+d27dwfO3r9/33z27du3zfmmTZua82E/QzCtuq5b9gvGyQ4hxA4hxA4hxA4hxA4hxA4hxA4hpub97BcuXGjOX79+3ZwfPHhwlOt8lWF3+Ldu3WrOnz59OnB2//79Fe00DufOnWvOt27d2py/evVqlOtMPSc7hBA7hBA7hBA7hBA7hBA7hJiaq7dhLl261PcK/ML8/Pyqnr958+aINsngZIcQYocQYocQYocQYocQYocQYocQMffsTJ+FhYW+V5goTnYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYIIXYI4f3srFm11uZ8z549zfla/nPVfXCyQwixQwixQwixQwixQwixQwhXb6xZXdc15zMzzqqv4bMFIcQOIcQOIcQOIcQOIcQOIcQOIdyzM7EOHDjQnF+7dm08i0wIJzuEEDuEEDuEEDuEEDuEEDuEEDuEcM/OmjXsV0nzdZzsEELsEELsEELsEELsEELsEELsEMI9O725fft2c37y5MkxbZLByQ4hxA4hxA4hxA4hxA4hxA4hxA4h6rC/gT3SF6t1fC8GobquW/YXATjZIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIYTYIcRYf5U00B8nO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4QQO4T4F4kOyXn94x1dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "urlretrieve(\"http://deeplearning.net/data/mnist/mnist.pkl.gz\",\n",
    "            \"mnist.pkl.gz\")\n",
    "with gzip.open(\"mnist.pkl.gz\") as f:\n",
    "    train_data, _, test_data = pickle.load(f, encoding=\"latin1\")\n",
    "train_data = list(train_data)\n",
    "test_data = list(test_data)\n",
    "for data in (train_data, test_data):\n",
    "    one_hot = np.zeros((data[0].shape[0], 10))\n",
    "    one_hot[np.arange(data[0].shape[0]), data[1]] = 1\n",
    "    data[1] = one_hot\n",
    "\n",
    "for i in range(3):\n",
    "    plt.figure()\n",
    "    plt.imshow(np.reshape(train_data[0][i], (28, 28)),\n",
    "               cmap=\"gray\")\n",
    "    plt.axis('off')\n",
    "    plt.title(str(np.argmax(train_data[1][i])));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with nengo.Network() as net:\n",
    "    # set some default parameters for the neurons that will make\n",
    "    # the training progress more smoothly\n",
    "    net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])\n",
    "    net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])\n",
    "    neuron_type = nengo.LIF(amplitude=0.01)\n",
    "\n",
    "    nengo_dl.configure_settings(trainable=False)\n",
    "\n",
    "    inp = nengo.Node(np.zeros(784))\n",
    "\n",
    "    temp = nengo.Probe(inp)\n",
    "\n",
    "    \n",
    "    #x0, conn0 = nengo_dl.tensor_layer(inp, neuron_type,\n",
    "    #                                  transform=nengo_dl.dists.Glorot(),\n",
    "    #                                 shape_in=(256,), return_conn=True)\n",
    "\n",
    "    x0 = nengo.Ensemble(784, 1, neuron_type = neuron_type)\n",
    "    weights_in = np.zeros((784,784))\n",
    "    for j in range(784):\n",
    "        weights_in[j][j] = 1\n",
    "    conn0 = nengo.Connection(inp, x0.neurons,transform=weights_in,synapse=None)\n",
    "    #net.config[x0].trainable = True\n",
    "    #net.config[conn0].trainable = True\n",
    "\n",
    "    x1, conn1 = nengo_dl.tensor_layer(x0, neuron_type,\n",
    "                                      transform=nengo_dl.dists.Glorot(),\n",
    "                                      shape_in=(64,), return_conn=True)\n",
    "\n",
    "    #net.config[x1].trainable = True\n",
    "    net.config[conn1].trainable = True\n",
    "\n",
    "    x = nengo.Node(size_in=10)\n",
    "    conn = nengo.Connection(x1, x, transform=nengo_dl.dists.Glorot(),\n",
    "                         synapse=None)\n",
    "\n",
    "    net.config[conn].trainable = True\n",
    "\n",
    "    # we'll create two different output probes, one with a filter\n",
    "    # (for when we're simulating the network over time and\n",
    "    # accumulating spikes), and one without (for when we're\n",
    "    # training the network using a rate-based approximation)\n",
    "    out_p = nengo.Probe(x)\n",
    "    out_p_filt = nengo.Probe(x, synapse=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build finished in 0:00:00                                                      \n",
      "|#                         Optimizing graph                           | 0:00:00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/nengo_dl/simulator.py:131: UserWarning: No GPU support detected. It is recommended that you install tensorflow-gpu (`pip install tensorflow-gpu`).\n",
      "  \"No GPU support detected. It is recommended that you \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization finished in 0:00:00                                               \n",
      "|#                        Constructing graph                          | 0:00:00WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Construction finished in 0:00:01                                               ##################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| ETA: 0:00:00\n"
     ]
    }
   ],
   "source": [
    "minibatch_size = 200\n",
    "sim = nengo_dl.Simulator(net, minibatch_size=minibatch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add the single timestep to the training data\n",
    "train_data = {inp: train_data[0][:, None, :],\n",
    "              out_p: train_data[1][:, None, :]}\n",
    "\n",
    "# when testing our network with spiking neurons we will need to run it\n",
    "# over time, so we repeat the input/target data for a number of\n",
    "# timesteps. we're also going to reduce the number of test images, just\n",
    "# to speed up this example.\n",
    "n_steps = 30\n",
    "test_data = {\n",
    "    inp: np.tile(test_data[0][:minibatch_size*2, None, :],\n",
    "                 (1, n_steps, 1)),\n",
    "    out_p_filt: np.tile(test_data[1][:minibatch_size*2, None, :],\n",
    "                        (1, n_steps, 1))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(outputs, targets):\n",
    "    return tf.nn.softmax_cross_entropy_with_logits_v2(\n",
    "        logits=outputs, labels=targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = tf.train.RMSPropOptimizer(learning_rate=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|####################Calculating loss (100%)###################| ETA:  00:00:00WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculation finished in 0:00:01                                                \n",
      "error before training: 91.75%\n"
     ]
    }
   ],
   "source": [
    "def classification_error(outputs, targets):\n",
    "    return 100 * tf.reduce_mean(\n",
    "        tf.cast(tf.not_equal(tf.argmax(outputs[:, -1], axis=-1),\n",
    "                             tf.argmax(targets[:, -1], axis=-1)),\n",
    "                tf.float32))\n",
    "\n",
    "\n",
    "print(\"error before training: %.2f%%\" % sim.loss(\n",
    "    test_data, {out_p_filt: classification_error}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|                   Training (0%)                  | ETA:  --:--:-- (loss: ---)WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training finished in 0:00:51 (loss: 15.5165)                                   \n"
     ]
    }
   ],
   "source": [
    "do_training = True\n",
    "if do_training:\n",
    "    # run training\n",
    "    sim.train(train_data, opt, objective={out_p: objective}, n_epochs=10)\n",
    "\n",
    "    # save the parameters to file\n",
    "    sim.save_params(\"./mnist_params_3l\")\n",
    "else:\n",
    "    # download pretrained weights\n",
    "    urlretrieve(\n",
    "        \"https://drive.google.com/uc?export=download&\"\n",
    "        \"id=1u9JyNuRxQDUcFgkRnI1qfJVFMdnGRsjI\",\n",
    "        \"mnist_params.zip\")\n",
    "    with zipfile.ZipFile(\"mnist_params.zip\") as f:\n",
    "        f.extractall()\n",
    "\n",
    "    # load parameters\n",
    "    sim.load_params(\"./mnist_params\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculation finished in 0:00:01                                                \n",
      "error after training: 4.00%\n"
     ]
    }
   ],
   "source": [
    "print(\"error after training: %.2f%%\" % sim.loss(\n",
    "    test_data, {out_p_filt: classification_error}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simulation finished in 0:00:01                                                 \n",
      "(784, 784)\n",
      "[[1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]]\n",
      "(64, 784)\n",
      "(10, 64)\n"
     ]
    }
   ],
   "source": [
    "sim.run_steps(n_steps, data={inp: test_data[inp][:minibatch_size]})\n",
    "\n",
    "for i in range(1):\n",
    "    print(sim.data[conn0].weights.shape) \n",
    "    print(sim.data[conn0].weights) \n",
    "    print(sim.data[conn1].weights.shape)\n",
    "    print(sim.data[conn].weights.shape)\n",
    "    with open('weights1.txt', 'w') as f:\n",
    "        print(sim.data[conn1].weights, file=f)\n",
    "    with open('weights2.txt', 'w') as fname:\n",
    "        print(sim.data[conn].weights, file=fname)\n",
    "        #print(sim.data[temp][i], file=f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sim.run_steps(n_steps, data={inp: test_data[inp][:minibatch_size]})\n",
    "\n",
    "for i in range(5):\n",
    "    plt.figure()\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.imshow(np.reshape(test_data[inp][i, 0], (28, 28)),\n",
    "               cmap=\"gray\")\n",
    "    plt.axis('off')\n",
    "\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.plot(sim.trange(), sim.data[out_p_filt][i])\n",
    "    plt.legend([str(i) for i in range(10)], loc=\"upper left\")\n",
    "    plt.xlabel(\"time\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}