Ch17 Futures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BD "
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'downloads/BD'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mFileNotFoundError\u001B[0m                         Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-3-5f1268d9df61>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m     43\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     44\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 45\u001B[1;33m \u001B[0mmain\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdownload_many\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32m<ipython-input-3-5f1268d9df61>\u001B[0m in \u001B[0;36mmain\u001B[1;34m(download_many)\u001B[0m\n\u001B[0;32m     38\u001B[0m \u001B[1;32mdef\u001B[0m \u001B[0mmain\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdownload_many\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     39\u001B[0m     \u001B[0mt0\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mperf_counter\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 40\u001B[1;33m     \u001B[0mcnt\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mdownload_many\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mPOP20_CC\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     41\u001B[0m     \u001B[0melapsed\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mperf_counter\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m-\u001B[0m \u001B[0mt0\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     42\u001B[0m     \u001B[0mmsg\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34mf'\\n{cnt} flags downloaded in {elapsed:.2f}s'\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m<ipython-input-3-5f1268d9df61>\u001B[0m in \u001B[0;36mdownload_many\u001B[1;34m(cc_list)\u001B[0m\n\u001B[0;32m     31\u001B[0m         \u001B[0mimg\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mget_flag\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mcc\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     32\u001B[0m         \u001B[0mshow\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mcc\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 33\u001B[1;33m         \u001B[0msave_flag\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mimg\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mcc\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     34\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     35\u001B[0m     \u001B[1;32mreturn\u001B[0m \u001B[0mlen\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mcc_list\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m<ipython-input-3-5f1268d9df61>\u001B[0m in \u001B[0;36msave_flag\u001B[1;34m(img, filename)\u001B[0m\n\u001B[0;32m     12\u001B[0m \u001B[1;32mdef\u001B[0m \u001B[0msave_flag\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mimg\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mfilename\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     13\u001B[0m     \u001B[0mpath\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mos\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mjoin\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mDEST_DIR\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mfilename\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 14\u001B[1;33m     \u001B[1;32mwith\u001B[0m \u001B[0mopen\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m'wb'\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mf\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     15\u001B[0m         \u001B[0mf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mwrite\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mimg\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     16\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: 'downloads/BD'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import sys\n",
    "import requests\n",
    "\n",
    "POP20_CC = ('CN IN US ID BR PK NG BD RU JP '\n",
    "            'MX PH VN ET EG DE IR TR CD FR').split()\n",
    "BASE_URL = 'http://flupy.org/data/flags'\n",
    "DEST_DIR = 'downloads/'\n",
    "\n",
    "\n",
    "def save_flag(img, filename):\n",
    "    path = os.path.join(DEST_DIR, filename)\n",
    "    with open(path, 'wb') as f:\n",
    "        f.write(img)\n",
    "\n",
    "\n",
    "def get_flag(cc):\n",
    "    url = f'{BASE_URL}/{cc.lower()}/{cc.lower()}.gif'\n",
    "    resp = requests.get(url)\n",
    "    return resp.content\n",
    "\n",
    "\n",
    "def show(text):\n",
    "    print(text, end=' ')\n",
    "    sys.stdout.flush()\n",
    "\n",
    "\n",
    "def download_many(cc_list):\n",
    "    for cc in sorted(cc_list):\n",
    "        img = get_flag(cc)\n",
    "        show(cc)\n",
    "        save_flag(img, cc)\n",
    "\n",
    "    return len(cc_list)\n",
    "\n",
    "\n",
    "def main(download_many):\n",
    "    t0 = time.perf_counter()\n",
    "    cnt = download_many(POP20_CC)\n",
    "    elapsed = time.perf_counter() - t0\n",
    "    msg = f'\\n{cnt} flags downloaded in {elapsed:.2f}s'\n",
    "\n",
    "\n",
    "main(download_many)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15:50:18]Script starting.\n",
      "[15:50:18]loiter(0): doing nothing for 0s...\n",
      "[15:50:18]loiter(0): done.\n",
      "[15:50:18]loiter(1): doing nothing for 1s...\n",
      "[15:50:18]loiter(2): doing nothing for 2s...\n",
      "[15:50:18]loiter(3): doing nothing for 3s...\n",
      "[15:50:18]results: <generator object Executor.map.<locals>.result_iterator at 0x000001C90A73FF20>\n",
      "[15:50:18]Waiting for individual results:\n",
      "[15:50:18]result 0: 0\n",
      "[15:50:19]loiter(1): done.\n",
      "[15:50:19]loiter(4): doing nothing for 4s...\n",
      "[15:50:19]result 1: 10\n",
      "[15:50:20]loiter(2): done.\n",
      "[15:50:20]result 2: 20\n",
      "[15:50:21]loiter(3): done.\n",
      "[15:50:21]result 3: 30\n",
      "[15:50:23]loiter(4): done.\n",
      "[15:50:23]result 4: 40\n",
      "[15:50:23]All done.\n"
     ]
    }
   ],
   "source": [
    "from time import strftime, sleep\n",
    "from concurrent import futures\n",
    "\n",
    "\n",
    "def display(*args):\n",
    "    print(strftime('[%H:%M:%S]'), end='')\n",
    "    print(*args)\n",
    "\n",
    "\n",
    "def loiter(n):\n",
    "    msg = '{}loiter({}): doing nothing for {}s...'\n",
    "    display(msg.format('', n, n))\n",
    "    sleep(n)\n",
    "    msg = '{}loiter({}): done.'\n",
    "    display(msg.format('', n))\n",
    "    return n * 10\n",
    "\n",
    "\n",
    "def main():\n",
    "    display('Script starting.')\n",
    "    executor = futures.ThreadPoolExecutor(max_workers=3)\n",
    "    results = executor.map(loiter, range(5))\n",
    "    display('results:', results)\n",
    "    display('Waiting for individual results:')\n",
    "    for i, result in enumerate(results):\n",
    "        display('result {}: {}'.format(i, result))\n",
    "    display('All done.')\n",
    "\n",
    "\n",
    "main()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{<Future at 0x218c7529e20 state=finished returned int>, <Future at 0x218c78bf2b0 state=finished returned int>, <Future at 0x218c7529cd0 state=finished returned int>, <Future at 0x218c84536d0 state=finished returned int>, <Future at 0x218c8440100 state=finished returned int>, <Future at 0x218c8453b50 state=finished returned int>, <Future at 0x218c8453370 state=finished returned int>, <Future at 0x218c8453580 state=finished returned int>, <Future at 0x218c7529190 state=finished returned int>, <Future at 0x218c84535e0 state=finished returned int>}\n"
     ]
    }
   ],
   "source": [
    "from concurrent import futures\n",
    "\n",
    "exec = futures.ThreadPoolExecutor(max_workers=5)\n",
    "\n",
    "\n",
    "def pprint(n):\n",
    "    return n * 2\n",
    "\n",
    "\n",
    "fs = [exec.submit(pprint, n) for n in range(10)]\n",
    "\n",
    "result = futures.wait(fs, return_when=futures.ALL_COMPLETED)\n",
    "print(result.done)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing d: 100%|██████████| 4/4 [00:02<00:00,  1.98it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "pbar = tqdm([\"a\", \"b\", \"c\", \"d\"])\n",
    "for char in pbar:\n",
    "    time.sleep(0.5)\n",
    "    pbar.set_description(\"Processing %s\" % char)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}