users/how-to/object-storage-minio-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c4ae4f29-ce08-4546-9d51-01d3d6b1dbc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xarray as xr\n",
    "import s3fs\n",
    "import zarr\n",
    "import numpy as np\n",
    "import dask.array as da\n",
    "import xarray as xr\n",
    "from dask.utils import parse_bytes\n",
    "import math\n",
    "import pandas as pd\n",
    "import dask\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "833332ac-c23f-4b48-8e91-12ad1b2b9917",
   "metadata": {},
   "source": [
    "# Simple MinIO access with s3fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a2ddfe89-cedc-4db4-8d27-a59a3cc71315",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!aws configure set aws_access_key_id xxx\n",
    "#!aws configure set aws_secret_access_key yyy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c660357b-4cff-4f07-8f01-7bcd743bea6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "access_key = !aws configure get aws_access_key_id\n",
    "access_key = access_key[0]\n",
    "secret_key = !aws configure get aws_secret_access_key\n",
    "secret_key = secret_key[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3f938612-2a10-48f0-97ef-f6ce5d14f8a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "client_kwargs={'endpoint_url': 'https://pangeo-eosc-minioapi.vm.fedcloud.eu/'}\n",
    "\n",
    "#s3 = s3fs.S3FileSystem(anon=False, client_kwargs=client_kwargs) # Works only when using s3 in this Notebook, not with distributed.\n",
    "s3 = s3fs.S3FileSystem(key=access_key, secret=secret_key, client_kwargs=client_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "03b164e0-eef5-4843-b614-2ee98db610cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['slunavalero-test/myfile',\n",
       " 'slunavalero-test/xarray-demo-dask-s3',\n",
       " 'slunavalero-test/xarray-demo-dask-s3-distributed']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3_prefix = os.environ['JUPYTERHUB_USER']\n",
    "# s3_suffix below is what you need to customise to create your own buckets:\n",
    "s3_suffix = 'test'\n",
    "s3_bucket = s3_prefix + '-' + s3_suffix + '/'\n",
    "s3.ls(s3_bucket)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f89c4de0-848c-4720-b35f-1d699435e7d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ResponseMetadata': {'RequestId': '1748E551D9273E59',\n",
       "  'HostId': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'date': 'Fri, 03 Mar 2023 11:30:00 GMT',\n",
       "   'content-length': '0',\n",
       "   'connection': 'keep-alive',\n",
       "   'accept-ranges': 'bytes',\n",
       "   'content-security-policy': 'block-all-mixed-content',\n",
       "   'etag': '\"d41d8cd98f00b204e9800998ecf8427e\"',\n",
       "   'strict-transport-security': 'max-age=15724800; includeSubDomains',\n",
       "   'vary': 'Accept-Encoding',\n",
       "   'x-amz-id-2': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',\n",
       "   'x-amz-request-id': '1748E551D9273E59',\n",
       "   'x-content-type-options': 'nosniff',\n",
       "   'x-xss-protection': '1; mode=block'},\n",
       "  'RetryAttempts': 0},\n",
       " 'ETag': '\"d41d8cd98f00b204e9800998ecf8427e\"'}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3.touch(s3_bucket + 'myfile')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e8f5cde6-5495-4353-be92-f848d3b42d9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['slunavalero-test/myfile',\n",
       " 'slunavalero-test/xarray-demo-dask-s3',\n",
       " 'slunavalero-test/xarray-demo-dask-s3-distributed']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3.ls(s3_bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8eb691f9-42ab-455e-912d-8a6c7ecdb87b",
   "metadata": {},
   "source": [
    "# Try to write some Zarr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e5c679e5-aa11-43ea-8aa5-2551192db872",
   "metadata": {},
   "outputs": [],
   "source": [
    "def timeseries(\n",
    "    chunk_per_worker=5,\n",
    "    chunk_size=\"128 MB\",\n",
    "    num_nodes=12,\n",
    "    worker_per_node=4,\n",
    "    chunking_scheme=None,\n",
    "    lat=320,\n",
    "    lon=384,\n",
    "    start=\"1980-01-01\",\n",
    "    freq=\"1H\",\n",
    "    nan=False,\n",
    "):\n",
    "    \"\"\" Create synthetic Xarray dataset filled with random\n",
    "    data.\n",
    "    Parameters\n",
    "    ----------\n",
    "    chunk_per_worker : int\n",
    "          number of chunk placed per worker.\n",
    "          see docs.dask.org, best practices, for chunk.\n",
    "          Best chunk size is around 100M but, each worker can\n",
    "          have many chunk, which automate the parallelism in dask.\n",
    "    chunk_size : str\n",
    "          chunk size in bytes, kilo, mega or any factor of bytes\n",
    "    num_nodes : int\n",
    "           number of compute nodes\n",
    "    worker_per_node: int\n",
    "           number of dask workers per node\n",
    "    chunking_scheme : str\n",
    "           Whether to chunk across time dimension ('temporal') or\n",
    "           horizontal dimensions (lat, lon) ('spatial').\n",
    "           If None, automatically determine chunk sizes along all dimensions.\n",
    "    lat : int\n",
    "         number of latitude values\n",
    "    lon : int\n",
    "         number of longitude values\n",
    "    start : datetime (or datetime-like string)\n",
    "        Start of time series\n",
    "    freq : string\n",
    "        String like '2s' or '1H' or '12W' for the time series frequency\n",
    "    nan : bool\n",
    "         Whether to include nan in generated data\n",
    "    Examples\n",
    "    ---------\n",
    "    >>> from benchmarks.datasets import timeseries\n",
    "    >>> ds = timeseries('128MB', 5, chunking_scheme='spatial', lat=500, lon=600)\n",
    "    >>> ds\n",
    "    <xarray.Dataset>\n",
    "    Dimensions:  (lat: 500, lon: 600, time: 267)\n",
    "    Coordinates:\n",
    "    * time     (time) datetime64[ns] 1980-01-01 1980-01-02 ... 1980-09-23\n",
    "    * lon      (lon) float64 -180.0 -179.4 -178.8 -178.2 ... 178.8 179.4 180.0\n",
    "    * lat      (lat) float64 -90.0 -89.64 -89.28 -88.92 ... 88.92 89.28 89.64 90.0\n",
    "    Data variables:\n",
    "        sst      (time, lon, lat) float64 dask.array<shape=(267, 600, 500), .....\n",
    "    Attributes:\n",
    "        history:  created for compute benchmarking\n",
    "    \"\"\"\n",
    "\n",
    "    dt = np.dtype(\"f8\")\n",
    "    itemsize = dt.itemsize\n",
    "    chunk_size = parse_bytes(chunk_size)\n",
    "    total_bytes = chunk_size * num_nodes * worker_per_node * chunk_per_worker\n",
    "    # total_bytes = chunk_size * num_nodes * worker_per_node\n",
    "    size = total_bytes / itemsize\n",
    "    timesteps = math.ceil(size / (lat * lon))\n",
    "    shape = (timesteps, lon, lat)\n",
    "    if chunking_scheme == \"temporal\":\n",
    "        x = math.ceil(chunk_size / (lon * lat * itemsize))\n",
    "        chunks = (x, lon, lat)\n",
    "    elif chunking_scheme == \"spatial\":\n",
    "        x = math.ceil(math.sqrt(chunk_size / (timesteps * itemsize)))\n",
    "        chunks = (timesteps, x, x)\n",
    "    else:\n",
    "        chunks = \"auto\"\n",
    "\n",
    "    lats = xr.DataArray(np.linspace(start=-90, stop=90, num=lat), dims=[\"lat\"])\n",
    "    lons = xr.DataArray(np.linspace(start=-180, stop=180, num=lon), dims=[\"lon\"])\n",
    "    times = xr.DataArray(pd.date_range(start=start, freq=freq, periods=timesteps), dims=[\"time\"])\n",
    "    if chunks == \"auto\":\n",
    "        with dask.config.set({\"array.chunk-size\": chunk_size}):\n",
    "            random_data = randn(shape=shape, chunks=chunks, nan=nan)\n",
    "    else:\n",
    "        random_data = randn(shape=shape, chunks=chunks, nan=nan)\n",
    "    ds = xr.DataArray(\n",
    "        random_data,\n",
    "        dims=[\"time\", \"lon\", \"lat\"],\n",
    "        coords={\"time\": times, \"lon\": lons, \"lat\": lats},\n",
    "        name=\"sst\",\n",
    "        attrs={\"units\": \"baz units\", \"description\": \"a description\"},\n",
    "    ).to_dataset()\n",
    "    ds.attrs = {\"history\": \"created for compute benchmarking\"}\n",
    "\n",
    "    return ds\n",
    "\n",
    "\n",
    "def randn(shape, chunks=None, nan=False, seed=0):\n",
    "    rng = da.random.RandomState(seed)\n",
    "    x = 5 + 3 * rng.standard_normal(shape, chunks=chunks)\n",
    "    if nan:\n",
    "        x = da.where(x < 0, np.nan, x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ec2058ae-b011-4686-bc28-a53d1f1e031d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
       "<defs>\n",
       "<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
       "<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
       "<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
       "<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
       "</symbol>\n",
       "<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
       "<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
       "<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
       "<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
       "<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
       "</symbol>\n",
       "</defs>\n",
       "</svg>\n",
       "<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
       " *\n",
       " */\n",
       "\n",
       ":root {\n",
       "  --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
       "  --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
       "  --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
       "  --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
       "  --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
       "  --xr-background-color: var(--jp-layout-color0, white);\n",
       "  --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
       "  --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
       "}\n",
       "\n",
       "html[theme=dark],\n",
       "body[data-theme=dark],\n",
       "body.vscode-dark {\n",
       "  --xr-font-color0: rgba(255, 255, 255, 1);\n",
       "  --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
       "  --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
       "  --xr-border-color: #1F1F1F;\n",
       "  --xr-disabled-color: #515151;\n",
       "  --xr-background-color: #111111;\n",
       "  --xr-background-color-row-even: #111111;\n",
       "  --xr-background-color-row-odd: #313131;\n",
       "}\n",
       "\n",
       ".xr-wrap {\n",
       "  display: block !important;\n",
       "  min-width: 300px;\n",
       "  max-width: 700px;\n",
       "}\n",
       "\n",
       ".xr-text-repr-fallback {\n",
       "  /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
       "  display: none;\n",
       "}\n",
       "\n",
       ".xr-header {\n",
       "  padding-top: 6px;\n",
       "  padding-bottom: 6px;\n",
       "  margin-bottom: 4px;\n",
       "  border-bottom: solid 1px var(--xr-border-color);\n",
       "}\n",
       "\n",
       ".xr-header > div,\n",
       ".xr-header > ul {\n",
       "  display: inline;\n",
       "  margin-top: 0;\n",
       "  margin-bottom: 0;\n",
       "}\n",
       "\n",
       ".xr-obj-type,\n",
       ".xr-array-name {\n",
       "  margin-left: 2px;\n",
       "  margin-right: 10px;\n",
       "}\n",
       "\n",
       ".xr-obj-type {\n",
       "  color: var(--xr-font-color2);\n",
       "}\n",
       "\n",
       ".xr-sections {\n",
       "  padding-left: 0 !important;\n",
       "  display: grid;\n",
       "  grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
       "}\n",
       "\n",
       ".xr-section-item {\n",
       "  display: contents;\n",
       "}\n",
       "\n",
       ".xr-section-item input {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       ".xr-section-item input + label {\n",
       "  color: var(--xr-disabled-color);\n",
       "}\n",
       "\n",
       ".xr-section-item input:enabled + label {\n",
       "  cursor: pointer;\n",
       "  color: var(--xr-font-color2);\n",
       "}\n",
       "\n",
       ".xr-section-item input:enabled + label:hover {\n",
       "  color: var(--xr-font-color0);\n",
       "}\n",
       "\n",
       ".xr-section-summary {\n",
       "  grid-column: 1;\n",
       "  color: var(--xr-font-color2);\n",
       "  font-weight: 500;\n",
       "}\n",
       "\n",
       ".xr-section-summary > span {\n",
       "  display: inline-block;\n",
       "  padding-left: 0.5em;\n",
       "}\n",
       "\n",
       ".xr-section-summary-in:disabled + label {\n",
       "  color: var(--xr-font-color2);\n",
       "}\n",
       "\n",
       ".xr-section-summary-in + label:before {\n",
       "  display: inline-block;\n",
       "  content: '►';\n",
       "  font-size: 11px;\n",
       "  width: 15px;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       ".xr-section-summary-in:disabled + label:before {\n",
       "  color: var(--xr-disabled-color);\n",
       "}\n",
       "\n",
       ".xr-section-summary-in:checked + label:before {\n",
       "  content: '▼';\n",
       "}\n",
       "\n",
       ".xr-section-summary-in:checked + label > span {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       ".xr-section-summary,\n",
       ".xr-section-inline-details {\n",
       "  padding-top: 4px;\n",
       "  padding-bottom: 4px;\n",
       "}\n",
       "\n",
       ".xr-section-inline-details {\n",
       "  grid-column: 2 / -1;\n",
       "}\n",
       "\n",
       ".xr-section-details {\n",
       "  display: none;\n",
       "  grid-column: 1 / -1;\n",
       "  margin-bottom: 5px;\n",
       "}\n",
       "\n",
       ".xr-section-summary-in:checked ~ .xr-section-details {\n",
       "  display: contents;\n",
       "}\n",
       "\n",
       ".xr-array-wrap {\n",
       "  grid-column: 1 / -1;\n",
       "  display: grid;\n",
       "  grid-template-columns: 20px auto;\n",
       "}\n",
       "\n",
       ".xr-array-wrap > label {\n",
       "  grid-column: 1;\n",
       "  vertical-align: top;\n",
       "}\n",
       "\n",
       ".xr-preview {\n",
       "  color: var(--xr-font-color3);\n",
       "}\n",
       "\n",
       ".xr-array-preview,\n",
       ".xr-array-data {\n",
       "  padding: 0 5px !important;\n",
       "  grid-column: 2;\n",
       "}\n",
       "\n",
       ".xr-array-data,\n",
       ".xr-array-in:checked ~ .xr-array-preview {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       ".xr-array-in:checked ~ .xr-array-data,\n",
       ".xr-array-preview {\n",
       "  display: inline-block;\n",
       "}\n",
       "\n",
       ".xr-dim-list {\n",
       "  display: inline-block !important;\n",
       "  list-style: none;\n",
       "  padding: 0 !important;\n",
       "  margin: 0;\n",
       "}\n",
       "\n",
       ".xr-dim-list li {\n",
       "  display: inline-block;\n",
       "  padding: 0;\n",
       "  margin: 0;\n",
       "}\n",
       "\n",
       ".xr-dim-list:before {\n",
       "  content: '(';\n",
       "}\n",
       "\n",
       ".xr-dim-list:after {\n",
       "  content: ')';\n",
       "}\n",
       "\n",
       ".xr-dim-list li:not(:last-child):after {\n",
       "  content: ',';\n",
       "  padding-right: 5px;\n",
       "}\n",
       "\n",
       ".xr-has-index {\n",
       "  font-weight: bold;\n",
       "}\n",
       "\n",
       ".xr-var-list,\n",
       ".xr-var-item {\n",
       "  display: contents;\n",
       "}\n",
       "\n",
       ".xr-var-item > div,\n",
       ".xr-var-item label,\n",
       ".xr-var-item > .xr-var-name span {\n",
       "  background-color: var(--xr-background-color-row-even);\n",
       "  margin-bottom: 0;\n",
       "}\n",
       "\n",
       ".xr-var-item > .xr-var-name:hover span {\n",
       "  padding-right: 5px;\n",
       "}\n",
       "\n",
       ".xr-var-list > li:nth-child(odd) > div,\n",
       ".xr-var-list > li:nth-child(odd) > label,\n",
       ".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
       "  background-color: var(--xr-background-color-row-odd);\n",
       "}\n",
       "\n",
       ".xr-var-name {\n",
       "  grid-column: 1;\n",
       "}\n",
       "\n",
       ".xr-var-dims {\n",
       "  grid-column: 2;\n",
       "}\n",
       "\n",
       ".xr-var-dtype {\n",
       "  grid-column: 3;\n",
       "  text-align: right;\n",
       "  color: var(--xr-font-color2);\n",
       "}\n",
       "\n",
       ".xr-var-preview {\n",
       "  grid-column: 4;\n",
       "}\n",
       "\n",
       ".xr-var-name,\n",
       ".xr-var-dims,\n",
       ".xr-var-dtype,\n",
       ".xr-preview,\n",
       ".xr-attrs dt {\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "  padding-right: 10px;\n",
       "}\n",
       "\n",
       ".xr-var-name:hover,\n",
       ".xr-var-dims:hover,\n",
       ".xr-var-dtype:hover,\n",
       ".xr-attrs dt:hover {\n",
       "  overflow: visible;\n",
       "  width: auto;\n",
       "  z-index: 1;\n",
       "}\n",
       "\n",
       ".xr-var-attrs,\n",
       ".xr-var-data {\n",
       "  display: none;\n",
       "  background-color: var(--xr-background-color) !important;\n",
       "  padding-bottom: 5px !important;\n",
       "}\n",
       "\n",
       ".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
       ".xr-var-data-in:checked ~ .xr-var-data {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       ".xr-var-data > table {\n",
       "  float: right;\n",
       "}\n",
       "\n",
       ".xr-var-name span,\n",
       ".xr-var-data,\n",
       ".xr-attrs {\n",
       "  padding-left: 25px !important;\n",
       "}\n",
       "\n",
       ".xr-attrs,\n",
       ".xr-var-attrs,\n",
       ".xr-var-data {\n",
       "  grid-column: 1 / -1;\n",
       "}\n",
       "\n",
       "dl.xr-attrs {\n",
       "  padding: 0;\n",
       "  margin: 0;\n",
       "  display: grid;\n",
       "  grid-template-columns: 125px auto;\n",
       "}\n",
       "\n",
       ".xr-attrs dt,\n",
       ".xr-attrs dd {\n",
       "  padding: 0;\n",
       "  margin: 0;\n",
       "  float: left;\n",
       "  padding-right: 10px;\n",
       "  width: auto;\n",
       "}\n",
       "\n",
       ".xr-attrs dt {\n",
       "  font-weight: normal;\n",
       "  grid-column: 1;\n",
       "}\n",
       "\n",
       ".xr-attrs dt:hover span {\n",
       "  display: inline-block;\n",
       "  background: var(--xr-background-color);\n",
       "  padding-right: 10px;\n",
       "}\n",
       "\n",
       ".xr-attrs dd {\n",
       "  grid-column: 2;\n",
       "  white-space: pre-wrap;\n",
       "  word-break: break-all;\n",
       "}\n",
       "\n",
       ".xr-icon-database,\n",
       ".xr-icon-file-text2 {\n",
       "  display: inline-block;\n",
       "  vertical-align: middle;\n",
       "  width: 1em;\n",
       "  height: 1.5em !important;\n",
       "  stroke-width: 0;\n",
       "  stroke: currentColor;\n",
       "  fill: currentColor;\n",
       "}\n",
       "</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
       "Dimensions:  (time: 69, lon: 384, lat: 320)\n",
       "Coordinates:\n",
       "  * time     (time) datetime64[ns] 1980-01-01 ... 1980-01-03T20:00:00\n",
       "  * lon      (lon) float64 -180.0 -179.1 -178.1 -177.2 ... 178.1 179.1 180.0\n",
       "  * lat      (lat) float64 -90.0 -89.44 -88.87 -88.31 ... 88.31 88.87 89.44 90.0\n",
       "Data variables:\n",
       "    sst      (time, lon, lat) float64 dask.array&lt;chunksize=(18, 384, 320), meta=np.ndarray&gt;\n",
       "Attributes:\n",
       "    history:  created for compute benchmarking</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-ee7717a8-8e49-4733-848a-b9232fb82314' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-ee7717a8-8e49-4733-848a-b9232fb82314' class='xr-section-summary'  title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 69</li><li><span class='xr-has-index'>lon</span>: 384</li><li><span class='xr-has-index'>lat</span>: 320</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-481a962f-7bcb-4814-9e5a-22ca0480f8cd' class='xr-section-summary-in' type='checkbox'  checked><label for='section-481a962f-7bcb-4814-9e5a-22ca0480f8cd' class='xr-section-summary' >Coordinates: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>1980-01-01 ... 1980-01-03T20:00:00</div><input id='attrs-e5d26259-2313-4b42-ab00-e628145f6d8d' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-e5d26259-2313-4b42-ab00-e628145f6d8d' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-b7908571-062f-43fc-a140-d85fb1045747' class='xr-var-data-in' type='checkbox'><label for='data-b7908571-062f-43fc-a140-d85fb1045747' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([&#x27;1980-01-01T00:00:00.000000000&#x27;, &#x27;1980-01-01T01:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T02:00:00.000000000&#x27;, &#x27;1980-01-01T03:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T04:00:00.000000000&#x27;, &#x27;1980-01-01T05:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T06:00:00.000000000&#x27;, &#x27;1980-01-01T07:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T08:00:00.000000000&#x27;, &#x27;1980-01-01T09:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T10:00:00.000000000&#x27;, &#x27;1980-01-01T11:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T12:00:00.000000000&#x27;, &#x27;1980-01-01T13:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T14:00:00.000000000&#x27;, &#x27;1980-01-01T15:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T16:00:00.000000000&#x27;, &#x27;1980-01-01T17:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T18:00:00.000000000&#x27;, &#x27;1980-01-01T19:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T20:00:00.000000000&#x27;, &#x27;1980-01-01T21:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-01T22:00:00.000000000&#x27;, &#x27;1980-01-01T23:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T00:00:00.000000000&#x27;, &#x27;1980-01-02T01:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T02:00:00.000000000&#x27;, &#x27;1980-01-02T03:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T04:00:00.000000000&#x27;, &#x27;1980-01-02T05:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T06:00:00.000000000&#x27;, &#x27;1980-01-02T07:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T08:00:00.000000000&#x27;, &#x27;1980-01-02T09:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T10:00:00.000000000&#x27;, &#x27;1980-01-02T11:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T12:00:00.000000000&#x27;, &#x27;1980-01-02T13:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T14:00:00.000000000&#x27;, &#x27;1980-01-02T15:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T16:00:00.000000000&#x27;, &#x27;1980-01-02T17:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T18:00:00.000000000&#x27;, &#x27;1980-01-02T19:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T20:00:00.000000000&#x27;, &#x27;1980-01-02T21:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-02T22:00:00.000000000&#x27;, &#x27;1980-01-02T23:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T00:00:00.000000000&#x27;, &#x27;1980-01-03T01:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T02:00:00.000000000&#x27;, &#x27;1980-01-03T03:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T04:00:00.000000000&#x27;, &#x27;1980-01-03T05:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T06:00:00.000000000&#x27;, &#x27;1980-01-03T07:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T08:00:00.000000000&#x27;, &#x27;1980-01-03T09:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T10:00:00.000000000&#x27;, &#x27;1980-01-03T11:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T12:00:00.000000000&#x27;, &#x27;1980-01-03T13:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T14:00:00.000000000&#x27;, &#x27;1980-01-03T15:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T16:00:00.000000000&#x27;, &#x27;1980-01-03T17:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T18:00:00.000000000&#x27;, &#x27;1980-01-03T19:00:00.000000000&#x27;,\n",
       "       &#x27;1980-01-03T20:00:00.000000000&#x27;], dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lon</span></div><div class='xr-var-dims'>(lon)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>-180.0 -179.1 ... 179.1 180.0</div><input id='attrs-eb3b2bf5-052e-4d11-97ab-3e30f6c8a251' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-eb3b2bf5-052e-4d11-97ab-3e30f6c8a251' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-74446a17-34dd-47f5-b132-4d418b03783d' class='xr-var-data-in' type='checkbox'><label for='data-74446a17-34dd-47f5-b132-4d418b03783d' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([-180.      , -179.060052, -178.120104, ...,  178.120104,  179.060052,\n",
       "        180.      ])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lat</span></div><div class='xr-var-dims'>(lat)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>-90.0 -89.44 -88.87 ... 89.44 90.0</div><input id='attrs-968bab14-09f1-4941-8d6d-6cc207292209' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-968bab14-09f1-4941-8d6d-6cc207292209' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-89fc9b38-9935-4598-8912-fea6210a73ad' class='xr-var-data-in' type='checkbox'><label for='data-89fc9b38-9935-4598-8912-fea6210a73ad' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([-90.      , -89.435737, -88.871473, ...,  88.871473,  89.435737,\n",
       "        90.      ])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-1f29db4c-b03f-4138-b07f-8a7850a1761c' class='xr-section-summary-in' type='checkbox'  checked><label for='section-1f29db4c-b03f-4138-b07f-8a7850a1761c' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>sst</span></div><div class='xr-var-dims'>(time, lon, lat)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array&lt;chunksize=(18, 384, 320), meta=np.ndarray&gt;</div><input id='attrs-9a315b5a-11c2-4ca4-8f12-29782814c312' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-9a315b5a-11c2-4ca4-8f12-29782814c312' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-fd3a5559-6e85-417e-ab67-049d7d7864e5' class='xr-var-data-in' type='checkbox'><label for='data-fd3a5559-6e85-417e-ab67-049d7d7864e5' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>units :</span></dt><dd>baz units</dd><dt><span>description :</span></dt><dd>a description</dd></dl></div><div class='xr-var-data'><table>\n",
       "    <tr>\n",
       "        <td>\n",
       "            <table>\n",
       "                <thead>\n",
       "                    <tr>\n",
       "                        <td> </td>\n",
       "                        <th> Array </th>\n",
       "                        <th> Chunk </th>\n",
       "                    </tr>\n",
       "                </thead>\n",
       "                <tbody>\n",
       "                    \n",
       "                    <tr>\n",
       "                        <th> Bytes </th>\n",
       "                        <td> 64.69 MiB </td>\n",
       "                        <td> 16.88 MiB </td>\n",
       "                    </tr>\n",
       "                    \n",
       "                    <tr>\n",
       "                        <th> Shape </th>\n",
       "                        <td> (69, 384, 320) </td>\n",
       "                        <td> (18, 384, 320) </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <th> Count </th>\n",
       "                        <td> 3 Graph Layers </td>\n",
       "                        <td> 4 Chunks </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                    <th> Type </th>\n",
       "                    <td> float64 </td>\n",
       "                    <td> numpy.ndarray </td>\n",
       "                    </tr>\n",
       "                </tbody>\n",
       "            </table>\n",
       "        </td>\n",
       "        <td>\n",
       "        <svg width=\"184\" height=\"194\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"34\" y2=\"24\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"10\" y1=\"120\" x2=\"34\" y2=\"144\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"120\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"16\" y1=\"6\" x2=\"16\" y2=\"126\" />\n",
       "  <line x1=\"22\" y1=\"12\" x2=\"22\" y2=\"132\" />\n",
       "  <line x1=\"29\" y1=\"19\" x2=\"29\" y2=\"139\" />\n",
       "  <line x1=\"34\" y1=\"24\" x2=\"34\" y2=\"144\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"10.0,0.0 34.43820742985945,24.438207429859453 34.43820742985945,144.43820742985946 10.0,120.0\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"110\" y2=\"0\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"16\" y1=\"6\" x2=\"116\" y2=\"6\" />\n",
       "  <line x1=\"22\" y1=\"12\" x2=\"122\" y2=\"12\" />\n",
       "  <line x1=\"29\" y1=\"19\" x2=\"129\" y2=\"19\" />\n",
       "  <line x1=\"34\" y1=\"24\" x2=\"134\" y2=\"24\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"10\" y1=\"0\" x2=\"34\" y2=\"24\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"110\" y1=\"0\" x2=\"134\" y2=\"24\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"10.0,0.0 110.0,0.0 134.43820742985946,24.438207429859453 34.43820742985945,24.438207429859453\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"34\" y1=\"24\" x2=\"134\" y2=\"24\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"34\" y1=\"144\" x2=\"134\" y2=\"144\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"34\" y1=\"24\" x2=\"34\" y2=\"144\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"134\" y1=\"24\" x2=\"134\" y2=\"144\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"34.43820742985945,24.438207429859453 134.43820742985946,24.438207429859453 134.43820742985946,144.43820742985946 34.43820742985945,144.43820742985946\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Text -->\n",
       "  <text x=\"84.438207\" y=\"164.438207\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >320</text>\n",
       "  <text x=\"154.438207\" y=\"84.438207\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,154.438207,84.438207)\">384</text>\n",
       "  <text x=\"12.219104\" y=\"152.219104\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,12.219104,152.219104)\">69</text>\n",
       "</svg>\n",
       "        </td>\n",
       "    </tr>\n",
       "</table></div></li></ul></div></li><li class='xr-section-item'><input id='section-87683d73-d6e5-4737-9562-6798b2e0f036' class='xr-section-summary-in' type='checkbox'  checked><label for='section-87683d73-d6e5-4737-9562-6798b2e0f036' class='xr-section-summary' >Attributes: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>history :</span></dt><dd>created for compute benchmarking</dd></dl></div></li></ul></div></div>"
      ],
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:  (time: 69, lon: 384, lat: 320)\n",
       "Coordinates:\n",
       "  * time     (time) datetime64[ns] 1980-01-01 ... 1980-01-03T20:00:00\n",
       "  * lon      (lon) float64 -180.0 -179.1 -178.1 -177.2 ... 178.1 179.1 180.0\n",
       "  * lat      (lat) float64 -90.0 -89.44 -88.87 -88.31 ... 88.31 88.87 89.44 90.0\n",
       "Data variables:\n",
       "    sst      (time, lon, lat) float64 dask.array<chunksize=(18, 384, 320), meta=np.ndarray>\n",
       "Attributes:\n",
       "    history:  created for compute benchmarking"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = timeseries(chunk_size='16 MiB', chunking_scheme='temporal', chunk_per_worker=1, num_nodes=1)\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6a4608cf-480d-4ac3-96c3-7f997b25a253",
   "metadata": {},
   "outputs": [],
   "source": [
    "store_s3 = s3fs.S3Map(root=s3_bucket + 'xarray-demo-dask-s3',\n",
    "                   s3=s3,\n",
    "                   check=False)\n",
    "store_s3.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ef1bf3a7-f95f-4688-8273-898defe36e2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 217 ms, sys: 32.2 ms, total: 249 ms\n",
      "Wall time: 1.29 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<xarray.backends.zarr.ZarrStore at 0x7f6b902154a0>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "ds.to_zarr(store=store_s3, mode='w', consolidated=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f601a971-585c-4305-8678-474095447a06",
   "metadata": {},
   "source": [
    "# Try some computation with dask-gateway"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "13b35228-608c-4fdf-aa14-d11f0883d60a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_gateway import Gateway\n",
    "gateway = Gateway()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6fd079e4-71b2-4a0a-a048-2507933bd5e8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/envs/notebook/lib/python3.9/site-packages/dask_gateway/client.py:674: GatewayWarning: Scale request of 6 workers would exceed resource limit of 4 workers. Scaling to 4 instead.\n",
      "  warnings.warn(GatewayWarning(msg[\"msg\"]))\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'VBox' object has no attribute '_ipython_display_'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/IPython/core/formatters.py:921\u001b[0m, in \u001b[0;36mIPythonDisplayFormatter.__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    919\u001b[0m method \u001b[38;5;241m=\u001b[39m get_real_method(obj, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprint_method)\n\u001b[1;32m    920\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m method \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 921\u001b[0m     \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    922\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/dask_gateway/client.py:1225\u001b[0m, in \u001b[0;36mGatewayCluster._ipython_display_\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m   1223\u001b[0m widget \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_widget()\n\u001b[1;32m   1224\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m widget \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1225\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwidget\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ipython_display_\u001b[49m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1226\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1227\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mIPython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdisplay\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m display\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'VBox' object has no attribute '_ipython_display_'"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
       "  <h3>GatewayCluster</h3>\n",
       "  <ul>\n",
       "    <li><b>Name: </b>c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8\n",
       "    <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8/status' target='_blank'>/services/dask-gateway/clusters/c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8/status</a>\n",
       "  </ul>\n",
       "</div>\n"
      ],
      "text/plain": [
       "GatewayCluster<c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8, status=running>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster = gateway.new_cluster(worker_memory=4)\n",
    "cluster.scale(6)\n",
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "90e9c26e-4cf6-482c-80e4-ebf69bd08564",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-c1f7ebd1-b9b6-11ed-839b-dee5c0c8ae27</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "\n",
       "        <tr>\n",
       "        \n",
       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
       "        \n",
       "        </tr>\n",
       "\n",
       "        \n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8/status\" target=\"_blank\">/services/dask-gateway/clusters/c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\"></td>\n",
       "            </tr>\n",
       "        \n",
       "\n",
       "        </table>\n",
       "\n",
       "        \n",
       "            <details>\n",
       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
       "            <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
       "  <h3>GatewayCluster</h3>\n",
       "  <ul>\n",
       "    <li><b>Name: </b>c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8\n",
       "    <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8/status' target='_blank'>/services/dask-gateway/clusters/c-scale-pangeo-dask.c58c80c2374847edac55dc1054895ab8/status</a>\n",
       "  </ul>\n",
       "</div>\n",
       "\n",
       "            </details>\n",
       "        \n",
       "\n",
       "    </div>\n",
       "</div>"
      ],
      "text/plain": [
       "<Client: 'tls://10.42.56.194:8786' processes=4 threads=4, memory=16.00 GiB>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dask.distributed import Client\n",
    "client = Client(cluster)\n",
    "client.wait_for_workers(4)\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eed6b684-41c3-4c52-b3f2-13a98f7fbe55",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import dask.array as da\n",
    "\n",
    "sample = 10_000_000_000  # <- this is huge!\n",
    "xxyy = da.random.uniform(-1, 1, size=(2, sample))\n",
    "norm = da.linalg.norm(xxyy, axis=0)\n",
    "summ = da.sum(norm <= 1)\n",
    "insiders = summ.compute()\n",
    "pi = 4 * insiders / sample\n",
    "print(\"pi ~= {}\".format(pi))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f98c3382-e079-41f7-9788-018c0f5d845a",
   "metadata": {},
   "source": [
    "# Try to write with distributed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bdb9b59-c738-4978-8092-647d04559c44",
   "metadata": {},
   "outputs": [],
   "source": [
    "store_s3 = s3fs.S3Map(root=s3_bucket + 'xarray-demo-dask-s3-distributed',\n",
    "                   s3=s3,\n",
    "                   check=False)\n",
    "store_s3.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66af75d1-afdf-4e78-8793-4afc163fbc55",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "ds.to_zarr(store=store_s3, mode='w', consolidated=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "027759bb-b2f4-497e-b069-a2ba0073d86d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}