mnist_cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import print_function\n",
    "\n",
    "import time\n",
    "import datetime\n",
    "import pytz\n",
    "\n",
    "import os\n",
    "\n",
    "from boto.s3.connection import S3Connection\n",
    "from boto.s3.key import Key\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers.convolutional import Convolution2D, MaxPooling2D\n",
    "from keras.callbacks import Callback\n",
    "from keras.utils import np_utils\n",
    "from keras.models import model_from_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def timestamp():\n",
    "    return int((datetime.datetime.now(tz=pytz.utc) - datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class S3ModelCheckpoint(Callback):\n",
    "    def __init__(self, bucket, monitor='val_loss', verbose=0):\n",
    "        super(Callback, self).__init__()\n",
    "        self.bucket = bucket\n",
    "        self.monitor = monitor\n",
    "        self.verbose = verbose\n",
    "        self.timestamp = timestamp()\n",
    "    \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        weights_key_name = 'weights_{timestamp}_{epoch:02d}_{val_loss:.2f}'.format(timestamp=self.timestamp, epoch=epoch, **logs)\n",
    "        model_key_name = 'model_{timestamp}_{epoch:02d}_{val_loss:.2f}'.format(timestamp=self.timestamp, epoch=epoch, **logs)\n",
    "\n",
    "        weights_filepath = '/tmp/{key_name}.hdf5'.format(key_name=weights_key_name)\n",
    "        \n",
    "        if self.verbose > 0:\n",
    "            print(\"Epoch {epoch:02d}: saving model to {model_key} and weights to {weights_key} inside {bucket_name}\".format(epoch=epoch, model_key=model_key_name, weights_key=weights_key_name, bucket_name=self.bucket.name))\n",
    "\n",
    "        model_key = Key(self.bucket)\n",
    "        model_key.key = model_key_name\n",
    "        model_key.set_contents_from_string(self.model.to_json())\n",
    "\n",
    "        self.model.save_weights(weights_filepath, overwrite=True)\n",
    "\n",
    "        weights_key = Key(self.bucket)\n",
    "        weights_key.key = weights_key_name\n",
    "        weights_key.set_contents_from_filename(weights_filepath)\n",
    "        \n",
    "        os.remove(weights_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "conn = S3Connection('AKIAJWIIMCGZFWQEVSAQ', 'OGJY/BgF+hxE2sFgTx1sMHRZiZCAbTVrFoMHrVru')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bucket_name = 'mnist_cnn'\n",
    "bucket = conn.lookup(bucket_name)\n",
    "if bucket is None:\n",
    "    bucket = conn.create_bucket(bucket_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Key: mnist_cnn,model_1445454184_00_0.03>\n",
      "<Key: mnist_cnn,model_1445454184_01_0.03>\n",
      "<Key: mnist_cnn,model_1445454184_02_0.03>\n",
      "<Key: mnist_cnn,model_1445456406_00_0.03>\n",
      "<Key: mnist_cnn,model_1445456406_01_0.03>\n",
      "<Key: mnist_cnn,model_1445456406_02_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_00_0.06>\n",
      "<Key: mnist_cnn,model_1445457932_01_0.04>\n",
      "<Key: mnist_cnn,model_1445457932_02_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_03_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_04_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_05_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_06_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_07_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_08_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_09_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_10_0.03>\n",
      "<Key: mnist_cnn,model_1445457932_11_0.03>\n",
      "<Key: mnist_cnn,weights_1445454184_00_0.03>\n",
      "<Key: mnist_cnn,weights_1445454184_01_0.03>\n",
      "<Key: mnist_cnn,weights_1445454184_02_0.03>\n",
      "<Key: mnist_cnn,weights_1445456406_00_0.03>\n",
      "<Key: mnist_cnn,weights_1445456406_01_0.03>\n",
      "<Key: mnist_cnn,weights_1445456406_02_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_00_0.06>\n",
      "<Key: mnist_cnn,weights_1445457932_01_0.04>\n",
      "<Key: mnist_cnn,weights_1445457932_02_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_03_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_04_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_05_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_06_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_07_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_08_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_09_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_10_0.03>\n",
      "<Key: mnist_cnn,weights_1445457932_11_0.03>\n"
     ]
    }
   ],
   "source": [
    "for key in bucket.list():\n",
    "    print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_key_name = 'model_1445457932_11_0.03'\n",
    "weights_key_name = 'weights_1445457932_11_0.03'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"layers\": [{\"b_constraint\": null, \"name\": \"Convolution2D\", \"activity_regularizer\": null, \"W_constraint\": null, \"input_shape\": [1, 28, 28], \"nb_col\": 3, \"subsample\": [1, 1], \"init\": \"glorot_uniform\", \"nb_filter\": 32, \"b_regularizer\": null, \"W_regularizer\": null, \"nb_row\": 3, \"activation\": \"linear\", \"border_mode\": \"full\"}, {\"beta\": 0.1, \"activation\": \"relu\", \"name\": \"Activation\", \"target\": 0}, {\"b_constraint\": null, \"name\": \"Convolution2D\", \"activity_regularizer\": null, \"W_constraint\": null, \"nb_col\": 3, \"subsample\": [1, 1], \"init\": \"glorot_uniform\", \"nb_filter\": 32, \"b_regularizer\": null, \"W_regularizer\": null, \"nb_row\": 3, \"activation\": \"linear\", \"border_mode\": \"valid\"}, {\"beta\": 0.1, \"activation\": \"relu\", \"name\": \"Activation\", \"target\": 0}, {\"stride\": [2, 2], \"name\": \"MaxPooling2D\", \"ignore_border\": true, \"pool_size\": [2, 2]}, {\"p\": 0.25, \"name\": \"Dropout\"}, {\"name\": \"Flatten\"}, {\"b_constraint\": null, \"name\": \"Dense\", \"activity_regularizer\": null, \"W_constraint\": null, \"init\": \"glorot_uniform\", \"activation\": \"linear\", \"input_dim\": null, \"b_regularizer\": null, \"W_regularizer\": null, \"output_dim\": 128}, {\"beta\": 0.1, \"activation\": \"relu\", \"name\": \"Activation\", \"target\": 0}, {\"p\": 0.5, \"name\": \"Dropout\"}, {\"b_constraint\": null, \"name\": \"Dense\", \"activity_regularizer\": null, \"W_constraint\": null, \"init\": \"glorot_uniform\", \"activation\": \"linear\", \"input_dim\": null, \"b_regularizer\": null, \"W_regularizer\": null, \"output_dim\": 10}, {\"beta\": 0.1, \"activation\": \"softmax\", \"name\": \"Activation\", \"target\": 0}], \"loss\": \"categorical_crossentropy\", \"theano_mode\": null, \"name\": \"Sequential\", \"class_mode\": \"categorical\", \"optimizer\": {\"epsilon\": 1e-06, \"lr\": 1.0, \"name\": \"Adadelta\", \"rho\": 0.95}}'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_key = bucket.get_key(model_key_name)\n",
    "model_json = model_key.get_contents_as_string()\n",
    "model_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "weights_key = bucket.get_key(weights_key_name)\n",
    "weights_key.get_contents_to_filename('/tmp/weights.hdf5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Train a simple convnet on the MNIST dataset.\n",
    "- Run on GPU: `THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python mnist_cnn.py`\n",
    "- Get to 99.25% test accuracy after 12 epochs (there is still a lot of margin for parameter tuning).\n",
    "- 16 seconds per epoch on a GRID K520 GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1337) # for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the data, shuffled and split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_classes = 10\n",
    "img_rows, img_cols = 28, 28 # input image dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (60000, 1, 28, 28)\n",
      "60000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)\n",
    "X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)\n",
    "X_train = X_train.astype(\"float32\")\n",
    "X_test = X_test.astype(\"float32\")\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "print('X_train shape:', X_train.shape)\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert class vectors to binary class matrices\n",
    "Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "Y_test = np_utils.to_categorical(y_test, nb_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Existing Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = model_from_json(model_json)\n",
    "model.load_weights('/tmp/weights.hdf5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# number of convolutional filters to use\n",
    "nb_filters = 32\n",
    "# size of pooling area for max pooling\n",
    "nb_pool = 2\n",
    "# convolution kernel size\n",
    "nb_conv = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv,\n",
    "                        border_mode='full',\n",
    "                        input_shape=(1, img_rows, img_cols)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Convolution2D(nb_filters, nb_conv, nb_conv))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(nb_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adadelta')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_epoch = 12\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "checkpoint = S3ModelCheckpoint(bucket, verbose=1)\n",
    "history = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_test, Y_test), callbacks=[checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 35s    \n",
      "Test score: 0.0257641568045\n",
      "Test accuracy: 0.9917\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=1)\n",
    "print('Test score:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "plt.plot(history.history['loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}