notebooks/cloud-data/blob-adls-numpy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read and write from Numpy to Azure Data Lake and Azure Blob storage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize public Blob filesystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "container_name = \"datasets\"\n",
    "storage_options = {\"account_name\": \"azuremlexamples\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from adlfs import AzureBlobFileSystem as abfs\n",
    "\n",
    "fs = abfs(**storage_options)\n",
    "files = fs.ls(f\"{container_name}/mnist\")\n",
    "files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define fuctions to read gzipped MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def read_images(f, num_images, image_size=28):\n",
    "    f.read(16)  # magic\n",
    "\n",
    "    buf = f.read(image_size * image_size * num_images)\n",
    "    images = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)\n",
    "    images = images.reshape(num_images, image_size, image_size, 1)\n",
    "\n",
    "    return images\n",
    "\n",
    "\n",
    "def read_labels(f, num_labels):\n",
    "    f.read(8)  # magic\n",
    "\n",
    "    buf = f.read(num_labels)\n",
    "    labels = np.frombuffer(buf, dtype=np.uint8)\n",
    "\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in gzipped MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "\n",
    "train_len = 60000\n",
    "test_len = 10000\n",
    "\n",
    "for f in files:\n",
    "    if \"train-images\" in f:\n",
    "        X_train = read_images(gzip.open(fs.open(f)), train_len)\n",
    "    elif \"train-labels\" in f:\n",
    "        y_train = read_labels(gzip.open(fs.open(f)), train_len)\n",
    "    elif \"images\" in f:\n",
    "        X_test = read_images(gzip.open(fs.open(f)), test_len)\n",
    "    elif \"labels\" in f:\n",
    "        y_test = read_labels(gzip.open(fs.open(f)), test_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify expected results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint\n",
    "\n",
    "i = randint(0, train_len)\n",
    "x = X_train[i]\n",
    "y = y_train[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.imshow(x.squeeze())\n",
    "plt.title(f\"Label: {y}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize private Blob filesystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Workspace\n",
    "\n",
    "ws = Workspace.from_config()\n",
    "ds = ws.datastores[\"workspaceblobstore\"]\n",
    "\n",
    "container_name = ds.container_name\n",
    "storage_options = {\"account_name\": ds.account_name, \"account_key\": ds.account_key}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fs = abfs(**storage_options)\n",
    "fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fs.ls(f\"{container_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write numpy arrays using `np.save`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with fs.open(f\"{container_name}/example-data/mnist/X_train.npy\", \"wb\") as f:\n",
    "    np.save(f, X_train)\n",
    "\n",
    "with fs.open(f\"{container_name}/example-data/mnist/y_train.npy\", \"wb\") as f:\n",
    "    np.save(f, y_train)\n",
    "\n",
    "with fs.open(f\"{container_name}/example-data/mnist/X_test.npy\", \"wb\") as f:\n",
    "    np.save(f, X_test)\n",
    "\n",
    "with fs.open(f\"{container_name}/example-data/mnist/y_test.npy\", \"wb\") as f:\n",
    "    np.save(f, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load numpy arrays using `np.load`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with fs.open(f\"{container_name}/example-data/mnist/X_train.npy\", \"rb\") as f:\n",
    "    X_train = np.load(f)\n",
    "\n",
    "with fs.open(f\"{container_name}/example-data/mnist/y_train.npy\", \"rb\") as f:\n",
    "    y_train = np.load(f)\n",
    "\n",
    "with fs.open(f\"{container_name}/example-data/mnist/X_test.npy\", \"rb\") as f:\n",
    "    X_test = np.load(f)\n",
    "\n",
    "with fs.open(f\"{container_name}/example-data/mnist/y_test.npy\", \"rb\") as f:\n",
    "    y_test = np.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify expected results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint\n",
    "\n",
    "i = randint(0, train_len)\n",
    "x = X_train[i]\n",
    "y = y_train[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.imshow(x.squeeze())\n",
    "plt.title(f\"Label: {y}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8 - AzureML",
   "language": "python",
   "name": "python38-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}