tiny_BERT_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tiny-BERT-training.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPwe2QLEKX9qVSrlgQmiP9m",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/menon92/DL-Sneak-Peek/blob/master/tiny_BERT_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3WNiDJ6mxpKw",
        "outputId": "be105ad1-1d63-45d0-d4c0-a26456956e52"
      },
      "source": [
        "!git clone https://github.com/google-research/bert.git\n",
        "!wget https://storage.googleapis.com/bert_models/2020_02_20/uncased_L-2_H-128_A-2.zip\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'bert'...\n",
            "remote: Enumerating objects: 340, done.\u001b[K\n",
            "remote: Total 340 (delta 0), reused 0 (delta 0), pack-reused 340\u001b[K\n",
            "Receiving objects: 100% (340/340), 328.28 KiB | 12.16 MiB/s, done.\n",
            "Resolving deltas: 100% (182/182), done.\n",
            "--2021-08-30 06:38:54--  https://storage.googleapis.com/bert_models/2020_02_20/uncased_L-2_H-128_A-2.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.167.128, 64.233.166.128, 74.125.133.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.167.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 16529104 (16M) [application/zip]\n",
            "Saving to: ‘uncased_L-2_H-128_A-2.zip’\n",
            "\n",
            "uncased_L-2_H-128_A 100%[===================>]  15.76M  18.2MB/s    in 0.9s    \n",
            "\n",
            "2021-08-30 06:38:56 (18.2 MB/s) - ‘uncased_L-2_H-128_A-2.zip’ saved [16529104/16529104]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NJGQWbnjx_5g",
        "outputId": "d333768a-2b2d-4dd1-8f90-e4aedc5f9501"
      },
      "source": [
        "!unzip uncased_L-2_H-128_A-2.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  uncased_L-2_H-128_A-2.zip\n",
            "  inflating: bert_model.ckpt.data-00000-of-00001  \n",
            "  inflating: bert_config.json        \n",
            "  inflating: vocab.txt               \n",
            "  inflating: bert_model.ckpt.index   \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GwGDAt_pyGdy",
        "outputId": "f47ad812-ce97-4b21-94fe-ca184043aa93"
      },
      "source": [
        "!du -sh uncased_L-2_H-128_A-2"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "du: cannot access 'uncased_L-2_H-128_A-2': No such file or directory\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qjc82ZadyJBj",
        "outputId": "81bc8266-29c2-4714-ba80-50e25e9c9d77"
      },
      "source": [
        "!pip install -qq tensorflow-gpu==1.15"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 411.5 MB 8.0 kB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 41.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 50 kB 6.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 503 kB 48.0 MB/s \n",
            "\u001b[?25h  Building wheel for gast (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.6.0 requires gast==0.4.0, but you have gast 0.2.2 which is incompatible.\n",
            "tensorflow 2.6.0 requires tensorboard~=2.6, but you have tensorboard 1.15.0 which is incompatible.\n",
            "tensorflow 2.6.0 requires tensorflow-estimator~=2.6, but you have tensorflow-estimator 1.15.1 which is incompatible.\n",
            "tensorflow-probability 0.13.0 requires gast>=0.3.2, but you have gast 0.2.2 which is incompatible.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WUmJM7Kfyz5s",
        "outputId": "6db76d67-ead8-4a51-da87-e9bd727fabe4"
      },
      "source": [
        "%cd bert/"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/bert\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jT-5Z7I9yijl",
        "outputId": "8e6214d5-869b-445e-caa9-14eabf1ca621"
      },
      "source": [
        "!python create_pretraining_data.py \\\n",
        "  --input_file=./sample_text.txt \\\n",
        "  --output_file=tf_examples.tfrecord \\\n",
        "  --vocab_file=/content/vocab.txt \\\n",
        "  --do_lower_case=True \\\n",
        "  --max_seq_length=128 \\\n",
        "  --max_predictions_per_seq=20 \\\n",
        "  --masked_lm_prob=0.15 \\\n",
        "  --random_seed=12345 \\\n",
        "  --dupe_factor=5"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From create_pretraining_data.py:469: The name tf.app.run is deprecated. Please use tf.compat.v1.app.run instead.\n",
            "\n",
            "WARNING:tensorflow:From create_pretraining_data.py:437: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
            "\n",
            "W0830 06:43:44.805232 140311880587136 module_wrapper.py:139] From create_pretraining_data.py:437: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
            "\n",
            "WARNING:tensorflow:From create_pretraining_data.py:437: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
            "\n",
            "W0830 06:43:44.805486 140311880587136 module_wrapper.py:139] From create_pretraining_data.py:437: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/bert/tokenization.py:125: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n",
            "W0830 06:43:44.805737 140311880587136 module_wrapper.py:139] From /content/bert/tokenization.py:125: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n",
            "WARNING:tensorflow:From create_pretraining_data.py:444: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
            "\n",
            "W0830 06:43:44.894430 140311880587136 module_wrapper.py:139] From create_pretraining_data.py:444: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
            "\n",
            "WARNING:tensorflow:From create_pretraining_data.py:446: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "W0830 06:43:44.895702 140311880587136 module_wrapper.py:139] From create_pretraining_data.py:446: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "INFO:tensorflow:*** Reading from input files ***\n",
            "I0830 06:43:44.895849 140311880587136 create_pretraining_data.py:446] *** Reading from input files ***\n",
            "INFO:tensorflow:  ./sample_text.txt\n",
            "I0830 06:43:44.896018 140311880587136 create_pretraining_data.py:448]   ./sample_text.txt\n",
            "INFO:tensorflow:*** Writing to output files ***\n",
            "I0830 06:43:44.944566 140311880587136 create_pretraining_data.py:457] *** Writing to output files ***\n",
            "INFO:tensorflow:  tf_examples.tfrecord\n",
            "I0830 06:43:44.944737 140311880587136 create_pretraining_data.py:459]   tf_examples.tfrecord\n",
            "WARNING:tensorflow:From create_pretraining_data.py:101: The name tf.python_io.TFRecordWriter is deprecated. Please use tf.io.TFRecordWriter instead.\n",
            "\n",
            "W0830 06:43:44.944976 140311880587136 module_wrapper.py:139] From create_pretraining_data.py:101: The name tf.python_io.TFRecordWriter is deprecated. Please use tf.io.TFRecordWriter instead.\n",
            "\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:44.945600 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] for more than a [MASK] up [MASK] great main street , [MASK] [MASK] [MASK] centre of the city , at right angles , [MASK] one equally magnificent , at each end ##ミ which , miles away , appeared , dim and distant over the heads of the living stream of passengers , the yellow [MASK] - hills of [MASK] [MASK] ; while at the end of the vista in front [MASK] them gleamed the blue harbour , through a network [SEP] possibly this may have been the reason why early rise ##rs in [MASK] locality , during the rainy season , adopted [MASK] [MASK] [MASK] of body , and seldom lifted [MASK] eyes [MASK] the rift [MASK] or india - ink washed skies above them . [SEP]\n",
            "I0830 06:43:44.945793 140311880587136 create_pretraining_data.py:151] tokens: [CLS] for more than a [MASK] up [MASK] great main street , [MASK] [MASK] [MASK] centre of the city , at right angles , [MASK] one equally magnificent , at each end ##ミ which , miles away , appeared , dim and distant over the heads of the living stream of passengers , the yellow [MASK] - hills of [MASK] [MASK] ; while at the end of the vista in front [MASK] them gleamed the blue harbour , through a network [SEP] possibly this may have been the reason why early rise ##rs in [MASK] locality , during the rainy season , adopted [MASK] [MASK] [MASK] of body , and seldom lifted [MASK] eyes [MASK] the rift [MASK] or india - ink washed skies above them . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2005 2062 2084 1037 103 2039 103 2307 2364 2395 1010 103 103 103 2803 1997 1996 2103 1010 2012 2157 12113 1010 103 2028 8053 12047 1010 2012 2169 2203 30250 2029 1010 2661 2185 1010 2596 1010 11737 1998 6802 2058 1996 4641 1997 1996 2542 5460 1997 5467 1010 1996 3756 103 1011 4564 1997 103 103 1025 2096 2012 1996 2203 1997 1996 13005 1999 2392 103 2068 25224 1996 2630 7440 1010 2083 1037 2897 102 4298 2023 2089 2031 2042 1996 3114 2339 2220 4125 2869 1999 103 10246 1010 2076 1996 16373 2161 1010 4233 103 103 103 1997 2303 1010 1998 15839 4196 103 2159 103 1996 16931 103 2030 2634 1011 10710 8871 15717 2682 2068 1012 102\n",
            "I0830 06:43:44.946022 140311880587136 create_pretraining_data.py:161] input_ids: 101 2005 2062 2084 1037 103 2039 103 2307 2364 2395 1010 103 103 103 2803 1997 1996 2103 1010 2012 2157 12113 1010 103 2028 8053 12047 1010 2012 2169 2203 30250 2029 1010 2661 2185 1010 2596 1010 11737 1998 6802 2058 1996 4641 1997 1996 2542 5460 1997 5467 1010 1996 3756 103 1011 4564 1997 103 103 1025 2096 2012 1996 2203 1997 1996 13005 1999 2392 103 2068 25224 1996 2630 7440 1010 2083 1037 2897 102 4298 2023 2089 2031 2042 1996 3114 2339 2220 4125 2869 1999 103 10246 1010 2076 1996 16373 2161 1010 4233 103 103 103 1997 2303 1010 1998 15839 4196 103 2159 103 1996 16931 103 2030 2634 1011 10710 8871 15717 2682 2068 1012 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:44.946218 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:44.946405 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 5 7 12 13 14 24 32 55 59 60 71 94 103 104 105 109 112 114 117 0\n",
            "I0830 06:43:44.946531 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 5 7 12 13 14 24 32 55 59 60 71 94 103 104 105 109 112 114 117 0\n",
            "INFO:tensorflow:masked_lm_ids: 3542 1996 4625 1999 1996 2011 1997 5472 1996 5532 1997 2008 1037 16465 10427 1998 2037 2000 2098 0\n",
            "I0830 06:43:44.946704 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 3542 1996 4625 1999 1996 2011 1997 5472 1996 5532 1997 2008 1037 16465 10427 1998 2037 2000 2098 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:44.946845 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:44.946992 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:44.947369 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] this text is included to make sure unicode is handled bracelet : 力 加 [MASK] 北 区 ᴵ ##ᴺ ##ᵀ ##ᵃ ##ছ [MASK] ##ট ##ড ##ণ ##ত [SEP] text should be one - [MASK] - per [MASK] line , with empty lines [MASK] documents . [SEP]\n",
            "I0830 06:43:44.947512 140311880587136 create_pretraining_data.py:151] tokens: [CLS] this text is included to make sure unicode is handled bracelet : 力 加 [MASK] 北 区 ᴵ ##ᴺ ##ᵀ ##ᵃ ##ছ [MASK] ##ট ##ড ##ণ ##ত [SEP] text should be one - [MASK] - per [MASK] line , with empty lines [MASK] documents . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2023 3793 2003 2443 2000 2191 2469 27260 2003 8971 19688 1024 1778 1779 103 1781 1782 1493 30030 30031 30032 29893 103 29895 29896 29897 29898 102 3793 2323 2022 2028 1011 103 1011 2566 103 2240 1010 2007 4064 3210 103 5491 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.947714 140311880587136 create_pretraining_data.py:161] input_ids: 101 2023 3793 2003 2443 2000 2191 2469 27260 2003 8971 19688 1024 1778 1779 103 1781 1782 1493 30030 30031 30032 29893 103 29895 29896 29897 29898 102 3793 2323 2022 2028 1011 103 1011 2566 103 2240 1010 2007 4064 3210 103 5491 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.947890 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.948097 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 11 15 23 34 37 38 43 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.948226 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 11 15 23 34 37 38 43 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 7919 1780 29894 6251 1011 2240 2090 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.948355 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 7919 1780 29894 6251 1011 2240 2090 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:44.948491 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:44.948609 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:44.949016 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] [MASK] from his wood pile [MASK] to [MASK] ##le a fire to dry his bed - [MASK] , and he had rec ##ours [MASK] [MASK] [MASK] more provide ##nt neighbor ' s to supply the deficiency . this was nearly opposite . [MASK] . cass [MASK] crossed the highway , and stopped suddenly . something glitter ##ed in the nearest red pool before [SEP] [MASK] [MASK] to relate , not an irregular , shape ##less fragment of crude ore , fresh from nature ' s cr ##ucible , but a [MASK] of jewel ##er ' [MASK] [MASK] ##ic ##raf ##t in the [MASK] of a [MASK] gold ring . looking at it more at ##ten ##tively , he saw that it bore the inscription , [MASK] [SEP]\n",
            "I0830 06:43:44.949229 140311880587136 create_pretraining_data.py:151] tokens: [CLS] [MASK] from his wood pile [MASK] to [MASK] ##le a fire to dry his bed - [MASK] , and he had rec ##ours [MASK] [MASK] [MASK] more provide ##nt neighbor ' s to supply the deficiency . this was nearly opposite . [MASK] . cass [MASK] crossed the highway , and stopped suddenly . something glitter ##ed in the nearest red pool before [SEP] [MASK] [MASK] to relate , not an irregular , shape ##less fragment of crude ore , fresh from nature ' s cr ##ucible , but a [MASK] of jewel ##er ' [MASK] [MASK] ##ic ##raf ##t in the [MASK] of a [MASK] gold ring . looking at it more at ##ten ##tively , he saw that it bore the inscription , [MASK] [SEP]\n",
            "INFO:tensorflow:input_ids: 101 103 2013 2010 3536 8632 103 2000 103 2571 1037 2543 2000 4318 2010 2793 1011 103 1010 1998 2002 2018 28667 22957 103 103 103 2062 3073 3372 11429 1005 1055 2000 4425 1996 18888 1012 2023 2001 3053 4500 1012 103 1012 16220 103 4625 1996 3307 1010 1998 3030 3402 1012 2242 27566 2098 1999 1996 7205 2417 4770 2077 102 103 103 2000 14396 1010 2025 2019 12052 1010 4338 3238 15778 1997 13587 10848 1010 4840 2013 3267 1005 1055 13675 21104 1010 2021 1037 103 1997 13713 2121 1005 103 103 2594 27528 2102 1999 1996 103 1997 1037 103 2751 3614 1012 2559 2012 2009 2062 2012 6528 25499 1010 2002 2387 2008 2009 8501 1996 9315 1010 103 102\n",
            "I0830 06:43:44.949450 140311880587136 create_pretraining_data.py:161] input_ids: 101 103 2013 2010 3536 8632 103 2000 103 2571 1037 2543 2000 4318 2010 2793 1011 103 1010 1998 2002 2018 28667 22957 103 103 103 2062 3073 3372 11429 1005 1055 2000 4425 1996 18888 1012 2023 2001 3053 4500 1012 103 1012 16220 103 4625 1996 3307 1010 1998 3030 3402 1012 2242 27566 2098 1999 1996 7205 2417 4770 2077 102 103 103 2000 14396 1010 2025 2019 12052 1010 4338 3238 15778 1997 13587 10848 1010 4840 2013 3267 1005 1055 13675 21104 1010 2021 1037 103 1997 13713 2121 1005 103 103 2594 27528 2102 1999 1996 103 1997 1037 103 2751 3614 1012 2559 2012 2009 2062 2012 6528 25499 1010 2002 2387 2008 2009 8501 1996 9315 1010 103 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:44.949670 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:44.949859 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 1 6 8 17 18 24 25 26 43 46 65 66 79 91 96 97 103 106 126 0\n",
            "I0830 06:43:44.950010 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 1 6 8 17 18 24 25 26 43 46 65 66 79 91 96 97 103 106 126 0\n",
            "INFO:tensorflow:masked_lm_ids: 11772 4188 2785 4253 1010 2063 2000 1037 2720 4173 1010 6919 10848 2978 1055 2192 2433 5810 1000 0\n",
            "I0830 06:43:44.950189 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 11772 4188 2785 4253 1010 2063 2000 1037 2720 4173 1010 6919 10848 2978 1055 2192 2433 5810 1000 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:44.950338 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:44.950484 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:44.950887 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] there [MASK] a phil vita ##phic pleasure in [MASK] one ' [MASK] treasures to the modest [MASK] . [SEP] looking [MASK] it more at ##ten ##tively , he saw that it [MASK] the inscription , \" may to cass . \" like most of his fellow gold [MASK] seekers [MASK] cass was super ##sti ##tious . [SEP]\n",
            "I0830 06:43:44.951077 140311880587136 create_pretraining_data.py:151] tokens: [CLS] there [MASK] a phil vita ##phic pleasure in [MASK] one ' [MASK] treasures to the modest [MASK] . [SEP] looking [MASK] it more at ##ten ##tively , he saw that it [MASK] the inscription , \" may to cass . \" like most of his fellow gold [MASK] seekers [MASK] cass was super ##sti ##tious . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2045 103 1037 6316 19300 17926 5165 1999 103 2028 1005 103 17605 2000 1996 10754 103 1012 102 2559 103 2009 2062 2012 6528 25499 1010 2002 2387 2008 2009 103 1996 9315 1010 1000 2089 2000 16220 1012 1000 2066 2087 1997 2010 3507 2751 103 24071 103 16220 2001 3565 16643 20771 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.951247 140311880587136 create_pretraining_data.py:161] input_ids: 101 2045 103 1037 6316 19300 17926 5165 1999 103 2028 1005 103 17605 2000 1996 10754 103 1012 102 2559 103 2009 2062 2012 6528 25499 1010 2002 2387 2008 2009 103 1996 9315 1010 1000 2089 2000 16220 1012 1000 2066 2087 1997 2010 3507 2751 103 24071 103 16220 2001 3565 16643 20771 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.951447 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:44.951637 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 2 5 9 12 17 21 32 48 50 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.034438 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 2 5 9 12 17 21 32 48 50 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 2003 19137 3098 1055 2402 2012 8501 1011 1010 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.034689 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2003 19137 3098 1055 2402 2012 8501 1011 1010 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.034883 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.035082 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.035804 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] at last they reached the [MASK] at taxi opposite end of the street ; [SEP] text should be one - sentence - per - line , with empty lines between documents . this sample text is public domain and [MASK] randomly selected [MASK] project [MASK] ##tenberg . [SEP]\n",
            "I0830 06:43:45.036044 140311880587136 create_pretraining_data.py:151] tokens: [CLS] at last they reached the [MASK] at taxi opposite end of the street ; [SEP] text should be one - sentence - per - line , with empty lines between documents . this sample text is public domain and [MASK] randomly selected [MASK] project [MASK] ##tenberg . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2012 2197 2027 2584 1996 103 2012 10095 4500 2203 1997 1996 2395 1025 102 3793 2323 2022 2028 1011 6251 1011 2566 1011 2240 1010 2007 4064 3210 2090 5491 1012 2023 7099 3793 2003 2270 5884 1998 103 18154 3479 103 2622 103 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.036327 140311880587136 create_pretraining_data.py:161] input_ids: 101 2012 2197 2027 2584 1996 103 2012 10095 4500 2203 1997 1996 2395 1025 102 3793 2323 2022 2028 1011 6251 1011 2566 1011 2240 1010 2007 4064 3210 2090 5491 1012 2023 7099 3793 2003 2270 5884 1998 103 18154 3479 103 2622 103 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.036588 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.036884 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 6 8 13 38 40 43 45 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.038214 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 6 8 13 38 40 43 45 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 21048 1996 2395 5884 2001 2013 9535 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.038401 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 21048 1996 2395 5884 2001 2013 9535 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.038607 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.038797 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.039525 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] text [MASK] ##stered one - sentence - per - [MASK] , with empty [MASK] between documents . [SEP] this sample text [MASK] public domain and was randomly selected from project gut ##tenberg . [SEP]\n",
            "I0830 06:43:45.039761 140311880587136 create_pretraining_data.py:151] tokens: [CLS] text [MASK] ##stered one - sentence - per - [MASK] , with empty [MASK] between documents . [SEP] this sample text [MASK] public domain and was randomly selected from project gut ##tenberg . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 3793 103 24167 2028 1011 6251 1011 2566 1011 103 1010 2007 4064 103 2090 5491 1012 102 2023 7099 3793 103 2270 5884 1998 2001 18154 3479 2013 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.040045 140311880587136 create_pretraining_data.py:161] input_ids: 101 3793 103 24167 2028 1011 6251 1011 2566 1011 103 1010 2007 4064 103 2090 5491 1012 102 2023 7099 3793 103 2270 5884 1998 2001 18154 3479 2013 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.040332 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.040567 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 2 3 10 14 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.040741 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 2 3 10 14 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 2323 2022 2240 3210 2003 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.040905 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2323 2022 2240 3210 2003 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.041096 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:45.041272 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.041900 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] the rain had only ceased [MASK] the gray streaks of morning [MASK] blazing star , and the settlement [MASK] to [MASK] moral [MASK] of clean ##liness , and the finding of forgotten knives , tin [MASK] [MASK] and smaller camp ut ##ens ##ils , where [MASK] heavy showers [MASK] washed ᵘ jonah debris and dust heap ##s [MASK] the cabin [MASK] [MASK] [SEP] [MASK] had once picked up on the highway a solid chunk of gold quartz which the rain subordinate freed from its inc ##umber ##ing soil , and washed into immediate and glittering popularity . possibly this may have been the [MASK] why early rise ##rs in [MASK] locality , during the rainy season , adopted a thoughtful habit [MASK] body , and seldom [SEP]\n",
            "I0830 06:43:45.042223 140311880587136 create_pretraining_data.py:151] tokens: [CLS] the rain had only ceased [MASK] the gray streaks of morning [MASK] blazing star , and the settlement [MASK] to [MASK] moral [MASK] of clean ##liness , and the finding of forgotten knives , tin [MASK] [MASK] and smaller camp ut ##ens ##ils , where [MASK] heavy showers [MASK] washed ᵘ jonah debris and dust heap ##s [MASK] the cabin [MASK] [MASK] [SEP] [MASK] had once picked up on the highway a solid chunk of gold quartz which the rain subordinate freed from its inc ##umber ##ing soil , and washed into immediate and glittering popularity . possibly this may have been the [MASK] why early rise ##rs in [MASK] locality , during the rainy season , adopted a thoughtful habit [MASK] body , and seldom [SEP]\n",
            "INFO:tensorflow:input_ids: 101 1996 4542 2018 2069 7024 103 1996 3897 21295 1997 2851 103 17162 2732 1010 1998 1996 4093 103 2000 103 7191 103 1997 4550 20942 1010 1998 1996 4531 1997 6404 13227 1010 9543 103 103 1998 3760 3409 21183 6132 12146 1010 2073 103 3082 23442 103 8871 1506 15617 11385 1998 6497 16721 2015 103 1996 6644 103 103 102 103 2018 2320 3856 2039 2006 1996 3307 1037 5024 20000 1997 2751 20971 2029 1996 4542 15144 10650 2013 2049 4297 29440 2075 5800 1010 1998 8871 2046 6234 1998 20332 6217 1012 4298 2023 2089 2031 2042 1996 103 2339 2220 4125 2869 1999 103 10246 1010 2076 1996 16373 2161 1010 4233 1037 16465 10427 103 2303 1010 1998 15839 102\n",
            "I0830 06:43:45.042537 140311880587136 create_pretraining_data.py:161] input_ids: 101 1996 4542 2018 2069 7024 103 1996 3897 21295 1997 2851 103 17162 2732 1010 1998 1996 4093 103 2000 103 7191 103 1997 4550 20942 1010 1998 1996 4531 1997 6404 13227 1010 9543 103 103 1998 3760 3409 21183 6132 12146 1010 2073 103 3082 23442 103 8871 1506 15617 11385 1998 6497 16721 2015 103 1996 6644 103 103 102 103 2018 2320 3856 2039 2006 1996 3307 1037 5024 20000 1997 2751 20971 2029 1996 4542 15144 10650 2013 2049 4297 29440 2075 5800 1010 1998 8871 2046 6234 1998 20332 6217 1012 4298 2023 2089 2031 2042 1996 103 2339 2220 4125 2869 1999 103 10246 1010 2076 1996 16373 2161 1010 4233 1037 16465 10427 103 2303 1010 1998 15839 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.042814 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.043795 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 6 12 19 21 23 36 37 46 49 51 52 58 61 62 64 81 104 110 122 0\n",
            "I0830 06:43:45.044046 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 6 12 19 21 23 36 37 46 49 51 52 58 61 62 64 81 104 110 122 0\n",
            "INFO:tensorflow:masked_lm_ids: 2007 2012 19179 1037 3168 10268 1010 1996 2018 2185 1996 2077 4303 1012 2099 2018 3114 2008 1997 0\n",
            "I0830 06:43:45.044202 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2007 2012 19179 1037 3168 10268 1010 1996 2018 2185 1996 2077 4303 1012 2099 2018 3114 2008 1997 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.044348 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:45.044481 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.044879 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] as the ancient sage - - the name is un ##im ##port ##ant to [MASK] [MASK] - - [MASK] [MASK] nightly [MASK] he might study by day , so i , the guardian of cloak [MASK] [MASK] [MASK] ##sol ##s [MASK] at the sacred doors of her lecture - [MASK] , im ##bib ##e celestial knowledge . [MASK] [MASK] youth i felt in me a soul above the matter - en ##tangled herd . she revealed to me the glorious [SEP] possibly this may have been the [MASK] [MASK] early rise ##rs in that locality , during the rainy [MASK] , adopted a thoughtful habit of body , and [MASK] lifted their eyes to the rift ##ed or india - ink washed skies above them . [SEP]\n",
            "I0830 06:43:45.045066 140311880587136 create_pretraining_data.py:151] tokens: [CLS] as the ancient sage - - the name is un ##im ##port ##ant to [MASK] [MASK] - - [MASK] [MASK] nightly [MASK] he might study by day , so i , the guardian of cloak [MASK] [MASK] [MASK] ##sol ##s [MASK] at the sacred doors of her lecture - [MASK] , im ##bib ##e celestial knowledge . [MASK] [MASK] youth i felt in me a soul above the matter - en ##tangled herd . she revealed to me the glorious [SEP] possibly this may have been the [MASK] [MASK] early rise ##rs in that locality , during the rainy [MASK] , adopted a thoughtful habit of body , and [MASK] lifted their eyes to the rift ##ed or india - ink washed skies above them . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2004 1996 3418 10878 1011 1011 1996 2171 2003 4895 5714 6442 4630 2000 103 103 1011 1011 103 103 22390 103 2002 2453 2817 2011 2154 1010 2061 1045 1010 1996 6697 1997 11965 103 103 103 19454 2015 103 2012 1996 6730 4303 1997 2014 8835 1011 103 1010 10047 28065 2063 17617 3716 1012 103 103 3360 1045 2371 1999 2033 1037 3969 2682 1996 3043 1011 4372 27898 14906 1012 2016 3936 2000 2033 1996 14013 102 4298 2023 2089 2031 2042 1996 103 103 2220 4125 2869 1999 2008 10246 1010 2076 1996 16373 103 1010 4233 1037 16465 10427 1997 2303 1010 1998 103 4196 2037 2159 2000 1996 16931 2098 2030 2634 1011 10710 8871 15717 2682 2068 1012 102\n",
            "I0830 06:43:45.045258 140311880587136 create_pretraining_data.py:161] input_ids: 101 2004 1996 3418 10878 1011 1011 1996 2171 2003 4895 5714 6442 4630 2000 103 103 1011 1011 103 103 22390 103 2002 2453 2817 2011 2154 1010 2061 1045 1010 1996 6697 1997 11965 103 103 103 19454 2015 103 2012 1996 6730 4303 1997 2014 8835 1011 103 1010 10047 28065 2063 17617 3716 1012 103 103 3360 1045 2371 1999 2033 1037 3969 2682 1996 3043 1011 4372 27898 14906 1012 2016 3936 2000 2033 1996 14013 102 4298 2023 2089 2031 2042 1996 103 103 2220 4125 2869 1999 2008 10246 1010 2076 1996 16373 103 1010 4233 1037 16465 10427 1997 2303 1010 1998 103 4196 2037 2159 2000 1996 16931 2098 2030 2634 1011 10710 8871 15717 2682 2068 1012 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.045448 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.045626 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 15 16 19 20 22 24 36 37 38 41 50 58 59 74 86 88 89 100 110 0\n",
            "I0830 06:43:45.045786 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 15 16 19 20 22 24 36 37 38 41 50 58 59 74 86 88 89 100 110 0\n",
            "INFO:tensorflow:masked_lm_ids: 1037 8284 16486 2300 2008 2453 2015 1998 11498 1010 2282 2013 2026 1012 2042 3114 2339 2161 15839 0\n",
            "I0830 06:43:45.045935 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 1037 8284 16486 2300 2008 2453 2015 1998 11498 1010 2282 2013 2026 1012 2042 3114 2339 2161 15839 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.046122 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.046241 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.046647 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] the rain had only ceased with the gray streaks of morning at blazing star , and the [MASK] awoke to a moral [MASK] of [MASK] ##liness , and the finding of forgotten knives [MASK] tin [MASK] , and smaller camp ut ##ens ##ils , where the [MASK] showers had washed away the debris and dust [MASK] ##s before the [MASK] doors . [SEP] this sample ##sily is public domain and was randomly selected [MASK] project gut ##tenberg . [SEP]\n",
            "I0830 06:43:45.046810 140311880587136 create_pretraining_data.py:151] tokens: [CLS] the rain had only ceased with the gray streaks of morning at blazing star , and the [MASK] awoke to a moral [MASK] of [MASK] ##liness , and the finding of forgotten knives [MASK] tin [MASK] , and smaller camp ut ##ens ##ils , where the [MASK] showers had washed away the debris and dust [MASK] ##s before the [MASK] doors . [SEP] this sample ##sily is public domain and was randomly selected [MASK] project gut ##tenberg . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 1996 4542 2018 2069 7024 2007 1996 3897 21295 1997 2851 2012 17162 2732 1010 1998 1996 103 19179 2000 1037 7191 103 1997 103 20942 1010 1998 1996 4531 1997 6404 13227 103 9543 103 1010 1998 3760 3409 21183 6132 12146 1010 2073 1996 103 23442 2018 8871 2185 1996 11385 1998 6497 103 2015 2077 1996 103 4303 1012 102 2023 7099 26863 2003 2270 5884 1998 2001 18154 3479 103 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.047007 140311880587136 create_pretraining_data.py:161] input_ids: 101 1996 4542 2018 2069 7024 2007 1996 3897 21295 1997 2851 2012 17162 2732 1010 1998 1996 103 19179 2000 1037 7191 103 1997 103 20942 1010 1998 1996 4531 1997 6404 13227 103 9543 103 1010 1998 3760 3409 21183 6132 12146 1010 2073 1996 103 23442 2018 8871 2185 1996 11385 1998 6497 103 2015 2077 1996 103 4303 1012 102 2023 7099 26863 2003 2270 5884 1998 2001 18154 3479 103 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.047189 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.047363 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 8 18 23 25 34 36 47 56 60 66 74 78 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.047528 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 8 18 23 25 34 36 47 56 60 66 74 78 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 3897 4093 3168 4550 1010 10268 3082 16721 6644 3793 2013 1012 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.047705 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 3897 4093 3168 4550 1010 10268 3082 16721 6644 3793 2013 1012 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.047860 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.048055 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.048455 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] this text is included to [MASK] sensual unicode is handled [MASK] : ##gp 加 勝 北 区 ᴵ ##ᴺ ##ᵀ ##ᵃ ##ছ ##জ ##ট ##ড ##ণ ##ত [SEP] roar and w ##hir ##l of the street , the perpetual stream of busy faces , the line of cu ##rri ##cles , pal ##an ##quin ##s , laden ass ##ᵐ , camel ##s , elephants [MASK] which met [MASK] passed him , and squeezed [MASK] up [MASK] and into doorway ##s , as they threaded their way through the great moon - gate [MASK] [MASK] [MASK] street beyond , drove everything from his mind [MASK] wondering curiosity , and a vague , helpless dread of that great living wilderness [MASK] more terrible than any dead wilderness of [SEP]\n",
            "I0830 06:43:45.048627 140311880587136 create_pretraining_data.py:151] tokens: [CLS] this text is included to [MASK] sensual unicode is handled [MASK] : ##gp 加 勝 北 区 ᴵ ##ᴺ ##ᵀ ##ᵃ ##ছ ##জ ##ট ##ড ##ণ ##ত [SEP] roar and w ##hir ##l of the street , the perpetual stream of busy faces , the line of cu ##rri ##cles , pal ##an ##quin ##s , laden ass ##ᵐ , camel ##s , elephants [MASK] which met [MASK] passed him , and squeezed [MASK] up [MASK] and into doorway ##s , as they threaded their way through the great moon - gate [MASK] [MASK] [MASK] street beyond , drove everything from his mind [MASK] wondering curiosity , and a vague , helpless dread of that great living wilderness [MASK] more terrible than any dead wilderness of [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2023 3793 2003 2443 2000 103 18753 27260 2003 8971 103 1024 21600 1779 1780 1781 1782 1493 30030 30031 30032 29893 29894 29895 29896 29897 29898 102 11950 1998 1059 11961 2140 1997 1996 2395 1010 1996 18870 5460 1997 5697 5344 1010 1996 2240 1997 12731 18752 18954 1010 14412 2319 12519 2015 1010 14887 4632 30038 1010 19130 2015 1010 16825 103 2029 2777 103 2979 2032 1010 1998 7757 103 2039 103 1998 2046 7086 2015 1010 2004 2027 26583 2037 2126 2083 1996 2307 4231 1011 4796 103 103 103 2395 3458 1010 5225 2673 2013 2010 2568 103 6603 10628 1010 1998 1037 13727 1010 13346 14436 1997 2008 2307 2542 9917 103 2062 6659 2084 2151 2757 9917 1997 102\n",
            "I0830 06:43:45.140809 140311880587136 create_pretraining_data.py:161] input_ids: 101 2023 3793 2003 2443 2000 103 18753 27260 2003 8971 103 1024 21600 1779 1780 1781 1782 1493 30030 30031 30032 29893 29894 29895 29896 29897 29898 102 11950 1998 1059 11961 2140 1997 1996 2395 1010 1996 18870 5460 1997 5697 5344 1010 1996 2240 1997 12731 18752 18954 1010 14412 2319 12519 2015 1010 14887 4632 30038 1010 19130 2015 1010 16825 103 2029 2777 103 2979 2032 1010 1998 7757 103 2039 103 1998 2046 7086 2015 1010 2004 2027 26583 2037 2126 2083 1996 2307 4231 1011 4796 103 103 103 2395 3458 1010 5225 2673 2013 2010 2568 103 6603 10628 1010 1998 1037 13727 1010 13346 14436 1997 2008 2307 2542 9917 103 2062 6659 2084 2151 2757 9917 1997 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.141115 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.141402 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 4 6 7 11 12 13 19 59 65 68 71 74 76 86 93 94 95 104 119 0\n",
            "I0830 06:43:45.141604 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 4 6 7 11 12 13 19 59 65 68 71 74 76 86 93 94 95 104 119 0\n",
            "INFO:tensorflow:masked_lm_ids: 2443 2191 2469 7919 1024 1778 30030 2229 1010 1998 1010 2032 4084 2126 2046 1996 20851 2021 1010 0\n",
            "I0830 06:43:45.141774 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2443 2191 2469 7919 1024 1778 30030 2229 1010 1998 1010 2032 4084 2126 2046 1996 20851 2021 1010 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.141965 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.142136 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.142920 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] and there burst on phil ##am ##mon ' [MASK] astonished eyes a vast semi ##ci ##rcle of blue sea , ring ##ed with palaces [MASK] ##berger . [SEP] [MASK] fortunate early rise rediscovered had once picked [MASK] [MASK] the highway a solid chunk of gold quartz which the rain had freed from its inc ##umber ##ing soil [MASK] and [MASK] into immediate and [MASK] popularity . possibly [MASK] may have been the reason why early rise ##rs [MASK] that locality , [MASK] the rainy season , adopted a thoughtful habit of body [MASK] and seldom lifted their eyes to the rift ##ed [MASK] india - ink washed skies above them [MASK] \" cass \" beard had risen early that morning [MASK] but not with a view [SEP]\n",
            "I0830 06:43:45.143173 140311880587136 create_pretraining_data.py:151] tokens: [CLS] and there burst on phil ##am ##mon ' [MASK] astonished eyes a vast semi ##ci ##rcle of blue sea , ring ##ed with palaces [MASK] ##berger . [SEP] [MASK] fortunate early rise rediscovered had once picked [MASK] [MASK] the highway a solid chunk of gold quartz which the rain had freed from its inc ##umber ##ing soil [MASK] and [MASK] into immediate and [MASK] popularity . possibly [MASK] may have been the reason why early rise ##rs [MASK] that locality , [MASK] the rainy season , adopted a thoughtful habit of body [MASK] and seldom lifted their eyes to the rift ##ed [MASK] india - ink washed skies above them [MASK] \" cass \" beard had risen early that morning [MASK] but not with a view [SEP]\n",
            "INFO:tensorflow:input_ids: 101 1998 2045 6532 2006 6316 3286 8202 1005 103 22741 2159 1037 6565 4100 6895 21769 1997 2630 2712 1010 3614 2098 2007 22763 103 14859 1012 102 103 19590 2220 4125 26733 2018 2320 3856 103 103 1996 3307 1037 5024 20000 1997 2751 20971 2029 1996 4542 2018 10650 2013 2049 4297 29440 2075 5800 103 1998 103 2046 6234 1998 103 6217 1012 4298 103 2089 2031 2042 1996 3114 2339 2220 4125 2869 103 2008 10246 1010 103 1996 16373 2161 1010 4233 1037 16465 10427 1997 2303 103 1998 15839 4196 2037 2159 2000 1996 16931 2098 103 2634 1011 10710 8871 15717 2682 2068 103 1000 16220 1000 10154 2018 13763 2220 2008 2851 103 2021 2025 2007 1037 3193 102\n",
            "I0830 06:43:45.143388 140311880587136 create_pretraining_data.py:161] input_ids: 101 1998 2045 6532 2006 6316 3286 8202 1005 103 22741 2159 1037 6565 4100 6895 21769 1997 2630 2712 1010 3614 2098 2007 22763 103 14859 1012 102 103 19590 2220 4125 26733 2018 2320 3856 103 103 1996 3307 1037 5024 20000 1997 2751 20971 2029 1996 4542 2018 10650 2013 2049 4297 29440 2075 5800 103 1998 103 2046 6234 1998 103 6217 1012 4298 103 2089 2031 2042 1996 3114 2339 2220 4125 2869 103 2008 10246 1010 103 1996 16373 2161 1010 4233 1037 16465 10427 1997 2303 103 1998 15839 4196 2037 2159 2000 1996 16931 2098 103 2634 1011 10710 8871 15717 2682 2068 103 1000 16220 1000 10154 2018 13763 2220 2008 2851 103 2021 2025 2007 1037 3193 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.143647 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.143836 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 9 25 26 29 33 37 38 56 58 60 64 68 78 82 93 103 111 113 121 0\n",
            "I0830 06:43:45.144028 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 9 25 26 29 33 37 38 56 58 60 64 68 78 82 93 103 111 113 121 0\n",
            "INFO:tensorflow:masked_lm_ids: 1055 1998 7626 1037 2099 2039 2006 2075 1010 8871 20332 2023 1999 2076 1010 2030 1012 16220 1010 0\n",
            "I0830 06:43:45.144181 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 1055 1998 7626 1037 2099 2039 2006 2075 1010 8871 20332 2023 1999 2076 1010 2030 1012 16220 1010 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.144323 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.144448 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.144884 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] like most of his fellow gold - seekers , cass was super ##sti ##tious . [SEP] at last they [MASK] the [MASK] at the opposite end of [MASK] street ; and there burst on phil ##am ##mon ' s astonished eyes a vast semi ##ci ##rcle of blue [MASK] , ring ##ed with palaces and towers . he stopped in ##vo ##lun ##tar ##ily ; and his [MASK] guide stopped also , and looked ask [MASK] at [MASK] [MASK] monk , to watch the effect [MASK] that [MASK] panorama should produce on him . [SEP]\n",
            "I0830 06:43:45.145061 140311880587136 create_pretraining_data.py:151] tokens: [CLS] like most of his fellow gold - seekers , cass was super ##sti ##tious . [SEP] at last they [MASK] the [MASK] at the opposite end of [MASK] street ; and there burst on phil ##am ##mon ' s astonished eyes a vast semi ##ci ##rcle of blue [MASK] , ring ##ed with palaces and towers . he stopped in ##vo ##lun ##tar ##ily ; and his [MASK] guide stopped also , and looked ask [MASK] at [MASK] [MASK] monk , to watch the effect [MASK] that [MASK] panorama should produce on him . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2066 2087 1997 2010 3507 2751 1011 24071 1010 16220 2001 3565 16643 20771 1012 102 2012 2197 2027 103 1996 103 2012 1996 4500 2203 1997 103 2395 1025 1998 2045 6532 2006 6316 3286 8202 1005 1055 22741 2159 1037 6565 4100 6895 21769 1997 2630 103 1010 3614 2098 2007 22763 1998 7626 1012 2002 3030 1999 6767 26896 7559 6588 1025 1998 2010 103 5009 3030 2036 1010 1998 2246 3198 103 2012 103 103 8284 1010 2000 3422 1996 3466 103 2008 103 23652 2323 3965 2006 2032 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.145258 140311880587136 create_pretraining_data.py:161] input_ids: 101 2066 2087 1997 2010 3507 2751 1011 24071 1010 16220 2001 3565 16643 20771 1012 102 2012 2197 2027 103 1996 103 2012 1996 4500 2203 1997 103 2395 1025 1998 2045 6532 2006 6316 3286 8202 1005 1055 22741 2159 1037 6565 4100 6895 21769 1997 2630 103 1010 3614 2098 2007 22763 1998 7626 1012 2002 3030 1999 6767 26896 7559 6588 1025 1998 2010 103 5009 3030 2036 1010 1998 2246 3198 103 2012 103 103 8284 1010 2000 3422 1996 3466 103 2008 103 23652 2323 3965 2006 2032 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.145452 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.145628 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 20 22 28 49 51 53 56 68 76 78 79 86 88 94 0 0 0 0 0 0\n",
            "I0830 06:43:45.145780 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 20 22 28 49 51 53 56 68 76 78 79 86 88 94 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 2584 21048 1996 2712 3614 2007 7626 2210 6651 1996 2402 2029 2882 1012 0 0 0 0 0 0\n",
            "I0830 06:43:45.145928 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2584 21048 1996 2712 3614 2007 7626 2210 6651 1996 2402 2029 2882 1012 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.146080 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.146209 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.146608 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] at last they reached the quay at the opposite end of [MASK] street ; and there burst on [MASK] ua [MASK] ' s astonished eyes a vast semi ##ci [MASK] of blue sea , ring ##ed with palaces and [MASK] . [SEP] this text is included to make sure [MASK] is handled properly : [MASK] [MASK] 勝 北 区 ᴵ ##ᴺ ##ᵀ ##ᵃ [MASK] ##জ ##ট ##ড ##ণ ##ত text should [MASK] one [MASK] sentence [MASK] per - line , with empty lines between 142 . this ##sable text is public domain and was randomly selected from project gut ##tenberg . [SEP]\n",
            "I0830 06:43:45.146808 140311880587136 create_pretraining_data.py:151] tokens: [CLS] at last they reached the quay at the opposite end of [MASK] street ; and there burst on [MASK] ua [MASK] ' s astonished eyes a vast semi ##ci [MASK] of blue sea , ring ##ed with palaces and [MASK] . [SEP] this text is included to make sure [MASK] is handled properly : [MASK] [MASK] 勝 北 区 ᴵ ##ᴺ ##ᵀ ##ᵃ [MASK] ##জ ##ট ##ড ##ণ ##ত text should [MASK] one [MASK] sentence [MASK] per - line , with empty lines between 142 . this ##sable text is public domain and was randomly selected from project gut ##tenberg . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2012 2197 2027 2584 1996 21048 2012 1996 4500 2203 1997 103 2395 1025 1998 2045 6532 2006 103 25423 103 1005 1055 22741 2159 1037 6565 4100 6895 103 1997 2630 2712 1010 3614 2098 2007 22763 1998 103 1012 102 2023 3793 2003 2443 2000 2191 2469 103 2003 8971 7919 1024 103 103 1780 1781 1782 1493 30030 30031 30032 103 29894 29895 29896 29897 29898 3793 2323 103 2028 103 6251 103 2566 1011 2240 1010 2007 4064 3210 2090 16087 1012 2023 19150 3793 2003 2270 5884 1998 2001 18154 3479 2013 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.147090 140311880587136 create_pretraining_data.py:161] input_ids: 101 2012 2197 2027 2584 1996 21048 2012 1996 4500 2203 1997 103 2395 1025 1998 2045 6532 2006 103 25423 103 1005 1055 22741 2159 1037 6565 4100 6895 103 1997 2630 2712 1010 3614 2098 2007 22763 1998 103 1012 102 2023 3793 2003 2443 2000 2191 2469 103 2003 8971 7919 1024 103 103 1780 1781 1782 1493 30030 30031 30032 103 29894 29895 29896 29897 29898 3793 2323 103 2028 103 6251 103 2566 1011 2240 1010 2007 4064 3210 2090 16087 1012 2023 19150 3793 2003 2270 5884 1998 2001 18154 3479 2013 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.147334 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.147544 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 12 19 20 21 30 40 50 55 56 64 72 74 76 85 88 0 0 0 0 0\n",
            "I0830 06:43:45.147706 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 12 19 20 21 30 40 50 55 56 64 72 74 76 85 88 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 1996 6316 3286 8202 21769 7626 27260 1778 1779 29893 2022 1011 1011 5491 7099 0 0 0 0 0\n",
            "I0830 06:43:45.147863 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 1996 6316 3286 8202 21769 7626 27260 1778 1779 29893 2022 1011 1011 5491 7099 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.148051 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.148176 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.148556 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] and there burst on phil ##am ##mon [MASK] s astonished eyes a vast [MASK] ##ci ##rcle of blue sea [MASK] ring ##ed with palaces and [MASK] . [SEP] he stopped in ##vo ##lun ##tar ##ily [MASK] and his little [MASK] stopped also , [MASK] looked ask ##ance at the young [MASK] , to watch the effect which that grand panorama persuasion produce on ##書 . [SEP]\n",
            "I0830 06:43:45.148715 140311880587136 create_pretraining_data.py:151] tokens: [CLS] and there burst on phil ##am ##mon [MASK] s astonished eyes a vast [MASK] ##ci ##rcle of blue sea [MASK] ring ##ed with palaces and [MASK] . [SEP] he stopped in ##vo ##lun ##tar ##ily [MASK] and his little [MASK] stopped also , [MASK] looked ask ##ance at the young [MASK] , to watch the effect which that grand panorama persuasion produce on ##書 . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 1998 2045 6532 2006 6316 3286 8202 103 1055 22741 2159 1037 6565 103 6895 21769 1997 2630 2712 103 3614 2098 2007 22763 1998 103 1012 102 2002 3030 1999 6767 26896 7559 6588 103 1998 2010 2210 103 3030 2036 1010 103 2246 3198 6651 2012 1996 2402 103 1010 2000 3422 1996 3466 2029 2008 2882 23652 27577 3965 2006 30397 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.244775 140311880587136 create_pretraining_data.py:161] input_ids: 101 1998 2045 6532 2006 6316 3286 8202 103 1055 22741 2159 1037 6565 103 6895 21769 1997 2630 2712 103 3614 2098 2007 22763 1998 103 1012 102 2002 3030 1999 6767 26896 7559 6588 103 1998 2010 2210 103 3030 2036 1010 103 2246 3198 6651 2012 1996 2402 103 1010 2000 3422 1996 3466 2029 2008 2882 23652 27577 3965 2006 30397 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.245081 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.245381 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 8 14 20 26 36 40 44 51 61 64 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.245572 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 8 14 20 26 36 40 44 51 61 64 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 1005 4100 1010 7626 1025 5009 1998 8284 2323 2032 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.245757 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 1005 4100 1010 7626 1025 5009 1998 8284 2323 2032 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.245964 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:45.246134 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.246725 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] [MASK] most of his fellow gold - [MASK] , cass was super ##sti ##tious . [SEP] while at the end of the vista in front [MASK] them gleamed the [MASK] harbour [MASK] through a network of beetles mast ##s . at [MASK] they reached the quay at the opposite end of the street ; and [MASK] burst [MASK] phil ##am ##mon ' s astonished eyes a vast semi ##ci ##rcle of blue sea ##ban [MASK] ##ed with palaces and towers . he stopped [MASK] ##vo [MASK] liege ##ily ; and his little guide stopped [MASK] , and looked ask ##ance at the young monk , to [MASK] the effect which that grand panorama should produce on [MASK] whoa [SEP]\n",
            "I0830 06:43:45.246965 140311880587136 create_pretraining_data.py:151] tokens: [CLS] [MASK] most of his fellow gold - [MASK] , cass was super ##sti ##tious . [SEP] while at the end of the vista in front [MASK] them gleamed the [MASK] harbour [MASK] through a network of beetles mast ##s . at [MASK] they reached the quay at the opposite end of the street ; and [MASK] burst [MASK] phil ##am ##mon ' s astonished eyes a vast semi ##ci ##rcle of blue sea ##ban [MASK] ##ed with palaces and towers . he stopped [MASK] ##vo [MASK] liege ##ily ; and his little guide stopped [MASK] , and looked ask ##ance at the young monk , to [MASK] the effect which that grand panorama should produce on [MASK] whoa [SEP]\n",
            "INFO:tensorflow:input_ids: 101 103 2087 1997 2010 3507 2751 1011 103 1010 16220 2001 3565 16643 20771 1012 102 2096 2012 1996 2203 1997 1996 13005 1999 2392 103 2068 25224 1996 103 7440 103 2083 1037 2897 1997 14538 15429 2015 1012 2012 103 2027 2584 1996 21048 2012 1996 4500 2203 1997 1996 2395 1025 1998 103 6532 103 6316 3286 8202 1005 1055 22741 2159 1037 6565 4100 6895 21769 1997 2630 2712 8193 103 2098 2007 22763 1998 7626 1012 2002 3030 103 6767 103 17766 6588 1025 1998 2010 2210 5009 3030 103 1010 1998 2246 3198 6651 2012 1996 2402 8284 1010 2000 103 1996 3466 2029 2008 2882 23652 2323 3965 2006 103 23281 102 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.247177 140311880587136 create_pretraining_data.py:161] input_ids: 101 103 2087 1997 2010 3507 2751 1011 103 1010 16220 2001 3565 16643 20771 1012 102 2096 2012 1996 2203 1997 1996 13005 1999 2392 103 2068 25224 1996 103 7440 103 2083 1037 2897 1997 14538 15429 2015 1012 2012 103 2027 2584 1996 21048 2012 1996 4500 2203 1997 1996 2395 1025 1998 103 6532 103 6316 3286 8202 1005 1055 22741 2159 1037 6565 4100 6895 21769 1997 2630 2712 8193 103 2098 2007 22763 1998 7626 1012 2002 3030 103 6767 103 17766 6588 1025 1998 2010 2210 5009 3030 103 1010 1998 2246 3198 6651 2012 1996 2402 8284 1010 2000 103 1996 3466 2029 2008 2882 23652 2323 3965 2006 103 23281 102 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.247387 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.247575 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 1 8 26 30 32 37 42 56 58 74 75 84 86 87 95 107 117 118 0 0\n",
            "I0830 06:43:45.247721 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 1 8 26 30 32 37 42 56 58 74 75 84 86 87 95 107 117 118 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 2066 24071 1997 2630 1010 14518 2197 2045 2006 1010 3614 1999 26896 7559 2036 3422 2032 1012 0 0\n",
            "I0830 06:43:45.247857 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2066 24071 1997 2630 1010 14518 2197 2045 2006 1010 3614 1999 26896 7559 2036 3422 2032 1012 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0\n",
            "I0830 06:43:45.248060 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.248204 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.249770 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] ! but , wonderful to relate , not an irregular , [MASK] ##less fragment of [MASK] [MASK] , fresh from nature ' s cr [MASK] , but a [MASK] of jewel ##er ' s hand ##ic ##raf ##t in the form of a plain gold [MASK] . looking at it [MASK] [MASK] [MASK] ##tively , he saw that [MASK] bore the inscription , [SEP] [MASK] on [MASK] more than a mile up the great main [MASK] , crossed in the centre of the city [MASK] at right angles , by one equally magnificent , at each end of which , miles away , [MASK] , dim and distant over [MASK] heads of the living stream golden passengers , the yellow [MASK] - hills of the desert [SEP]\n",
            "I0830 06:43:45.250030 140311880587136 create_pretraining_data.py:151] tokens: [CLS] ! but , wonderful to relate , not an irregular , [MASK] ##less fragment of [MASK] [MASK] , fresh from nature ' s cr [MASK] , but a [MASK] of jewel ##er ' s hand ##ic ##raf ##t in the form of a plain gold [MASK] . looking at it [MASK] [MASK] [MASK] ##tively , he saw that [MASK] bore the inscription , [SEP] [MASK] on [MASK] more than a mile up the great main [MASK] , crossed in the centre of the city [MASK] at right angles , by one equally magnificent , at each end of which , miles away , [MASK] , dim and distant over [MASK] heads of the living stream golden passengers , the yellow [MASK] - hills of the desert [SEP]\n",
            "INFO:tensorflow:input_ids: 101 999 2021 1010 6919 2000 14396 1010 2025 2019 12052 1010 103 3238 15778 1997 103 103 1010 4840 2013 3267 1005 1055 13675 103 1010 2021 1037 103 1997 13713 2121 1005 1055 2192 2594 27528 2102 1999 1996 2433 1997 1037 5810 2751 103 1012 2559 2012 2009 103 103 103 25499 1010 2002 2387 2008 103 8501 1996 9315 1010 102 103 2006 103 2062 2084 1037 3542 2039 1996 2307 2364 103 1010 4625 1999 1996 2803 1997 1996 2103 103 2012 2157 12113 1010 2011 2028 8053 12047 1010 2012 2169 2203 1997 2029 1010 2661 2185 1010 103 1010 11737 1998 6802 2058 103 4641 1997 1996 2542 5460 3585 5467 1010 1996 3756 103 1011 4564 1997 1996 5532 102\n",
            "I0830 06:43:45.250259 140311880587136 create_pretraining_data.py:161] input_ids: 101 999 2021 1010 6919 2000 14396 1010 2025 2019 12052 1010 103 3238 15778 1997 103 103 1010 4840 2013 3267 1005 1055 13675 103 1010 2021 1037 103 1997 13713 2121 1005 1055 2192 2594 27528 2102 1999 1996 2433 1997 1037 5810 2751 103 1012 2559 2012 2009 103 103 103 25499 1010 2002 2387 2008 103 8501 1996 9315 1010 102 103 2006 103 2062 2084 1037 3542 2039 1996 2307 2364 103 1010 4625 1999 1996 2803 1997 1996 2103 103 2012 2157 12113 1010 2011 2028 8053 12047 1010 2012 2169 2203 1997 2029 1010 2661 2185 1010 103 1010 11737 1998 6802 2058 103 4641 1997 1996 2542 5460 3585 5467 1010 1996 3756 103 1011 4564 1997 1996 5532 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.250472 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.250735 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 7 12 16 17 25 29 46 51 52 53 59 65 67 76 85 104 110 116 121 0\n",
            "I0830 06:43:45.250910 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 7 12 16 17 25 29 46 51 52 53 59 65 67 76 85 104 110 116 121 0\n",
            "INFO:tensorflow:masked_lm_ids: 1010 4338 13587 10848 21104 2978 3614 2062 2012 6528 2009 2218 2005 2395 1010 2596 1996 1997 5472 0\n",
            "I0830 06:43:45.251109 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 1010 4338 13587 10848 21104 2978 3614 2062 2012 6528 2009 2218 2005 2395 1010 2596 1996 1997 5472 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.251277 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.251410 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.252063 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] there is a phil ##oso ##phic pleasure in opening one ' s treasures to [MASK] modest young . perhaps you will assist me by carrying recording basket [MASK] [MASK] ? ' [MASK] the [MASK] man jumped up , put his basket on phil ##am myths ' s head , and tr ##otted off [MASK] a neighbouring street . [SEP] his ragged little api ##sh guide ; but the novel roar [MASK] w ##hir ##l [MASK] the street , the perpetual stream of busy faces [MASK] the line of cu ##rri ##cles [MASK] pal [MASK] ##quin ##s , laden ass ##es , camel ##s [MASK] elephants , which met [MASK] passed him , and [MASK] him up steps and into [MASK] ##s , as they threaded their [SEP]\n",
            "I0830 06:43:45.252295 140311880587136 create_pretraining_data.py:151] tokens: [CLS] there is a phil ##oso ##phic pleasure in opening one ' s treasures to [MASK] modest young . perhaps you will assist me by carrying recording basket [MASK] [MASK] ? ' [MASK] the [MASK] man jumped up , put his basket on phil ##am myths ' s head , and tr ##otted off [MASK] a neighbouring street . [SEP] his ragged little api ##sh guide ; but the novel roar [MASK] w ##hir ##l [MASK] the street , the perpetual stream of busy faces [MASK] the line of cu ##rri ##cles [MASK] pal [MASK] ##quin ##s , laden ass ##es , camel ##s [MASK] elephants , which met [MASK] passed him , and [MASK] him up steps and into [MASK] ##s , as they threaded their [SEP]\n",
            "INFO:tensorflow:input_ids: 101 2045 2003 1037 6316 19137 17926 5165 1999 3098 2028 1005 1055 17605 2000 103 10754 2402 1012 3383 2017 2097 6509 2033 2011 4755 3405 10810 103 103 1029 1005 103 1996 103 2158 5598 2039 1010 2404 2010 10810 2006 6316 3286 17218 1005 1055 2132 1010 1998 19817 26174 2125 103 1037 9632 2395 1012 102 2010 14202 2210 17928 4095 5009 1025 2021 1996 3117 11950 103 1059 11961 2140 103 1996 2395 1010 1996 18870 5460 1997 5697 5344 103 1996 2240 1997 12731 18752 18954 103 14412 103 12519 2015 1010 14887 4632 2229 1010 19130 2015 103 16825 1010 2029 2777 103 2979 2032 1010 1998 103 2032 2039 4084 1998 2046 103 2015 1010 2004 2027 26583 2037 102\n",
            "I0830 06:43:45.252549 140311880587136 create_pretraining_data.py:161] input_ids: 101 2045 2003 1037 6316 19137 17926 5165 1999 3098 2028 1005 1055 17605 2000 103 10754 2402 1012 3383 2017 2097 6509 2033 2011 4755 3405 10810 103 103 1029 1005 103 1996 103 2158 5598 2039 1010 2404 2010 10810 2006 6316 3286 17218 1005 1055 2132 1010 1998 19817 26174 2125 103 1037 9632 2395 1012 102 2010 14202 2210 17928 4095 5009 1025 2021 1996 3117 11950 103 1059 11961 2140 103 1996 2395 1010 1996 18870 5460 1997 5697 5344 103 1996 2240 1997 12731 18752 18954 103 14412 103 12519 2015 1010 14887 4632 2229 1010 19130 2015 103 16825 1010 2029 2777 103 2979 2032 1010 1998 103 2032 2039 4084 1998 2046 103 2015 1010 2004 2027 26583 2037 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.252826 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.253083 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 15 26 28 29 32 34 45 54 71 75 85 92 94 104 109 113 114 116 120 0\n",
            "I0830 06:43:45.253249 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 15 26 28 29 32 34 45 54 71 75 85 92 94 104 109 113 114 116 120 0\n",
            "INFO:tensorflow:masked_lm_ids: 1996 2023 1997 5909 1998 2210 8202 2039 1998 1997 1010 1010 2319 1010 1998 1998 7757 2039 7086 0\n",
            "I0830 06:43:45.343087 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 1996 2023 1997 5909 1998 2210 8202 2039 1998 1997 1010 1010 2319 1010 1998 1998 7757 2039 7086 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.343343 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:45.343521 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.344400 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] perhaps murder will assist me by carrying this [MASK] of fruit ? ' [MASK] the little man jumped up , put his basket [MASK] phil ##am ##mon ' [MASK] head , and tr ##otted off up a neighbouring street . phil ##am ##mon followed , half contempt ##uous , half wondering at what this philosophy [MASK] be , which [MASK] [MASK] the self [MASK] con ##ce ##it of anything so ab ##ject as his ragged [MASK] api ##val guide ; [SEP] text should be [MASK] [MASK] sentence - per [MASK] line [MASK] with empty lines [MASK] documents . this sample text is public domain and was randomly selected from project gut ##tenberg . [SEP]\n",
            "I0830 06:43:45.344646 140311880587136 create_pretraining_data.py:151] tokens: [CLS] perhaps murder will assist me by carrying this [MASK] of fruit ? ' [MASK] the little man jumped up , put his basket [MASK] phil ##am ##mon ' [MASK] head , and tr ##otted off up a neighbouring street . phil ##am ##mon followed , half contempt ##uous , half wondering at what this philosophy [MASK] be , which [MASK] [MASK] the self [MASK] con ##ce ##it of anything so ab ##ject as his ragged [MASK] api ##val guide ; [SEP] text should be [MASK] [MASK] sentence - per [MASK] line [MASK] with empty lines [MASK] documents . this sample text is public domain and was randomly selected from project gut ##tenberg . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 3383 4028 2097 6509 2033 2011 4755 2023 103 1997 5909 1029 1005 103 1996 2210 2158 5598 2039 1010 2404 2010 10810 103 6316 3286 8202 1005 103 2132 1010 1998 19817 26174 2125 2039 1037 9632 2395 1012 6316 3286 8202 2628 1010 2431 17152 8918 1010 2431 6603 2012 2054 2023 4695 103 2022 1010 2029 103 103 1996 2969 103 9530 3401 4183 1997 2505 2061 11113 20614 2004 2010 14202 103 17928 10175 5009 1025 102 3793 2323 2022 103 103 6251 1011 2566 103 2240 103 2007 4064 3210 103 5491 1012 2023 7099 3793 2003 2270 5884 1998 2001 18154 3479 2013 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.344974 140311880587136 create_pretraining_data.py:161] input_ids: 101 3383 4028 2097 6509 2033 2011 4755 2023 103 1997 5909 1029 1005 103 1996 2210 2158 5598 2039 1010 2404 2010 10810 103 6316 3286 8202 1005 103 2132 1010 1998 19817 26174 2125 2039 1037 9632 2395 1012 6316 3286 8202 2628 1010 2431 17152 8918 1010 2431 6603 2012 2054 2023 4695 103 2022 1010 2029 103 103 1996 2969 103 9530 3401 4183 1997 2505 2061 11113 20614 2004 2010 14202 103 17928 10175 5009 1025 102 3793 2323 2022 103 103 6251 1011 2566 103 2240 103 2007 4064 3210 103 5491 1012 2023 7099 3793 2003 2270 5884 1998 2001 18154 3479 2013 2622 9535 21806 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.345240 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.345483 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 2 9 14 24 29 56 60 61 63 64 76 78 85 86 90 92 96 0 0 0\n",
            "I0830 06:43:45.345684 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 2 9 14 24 29 56 60 61 63 64 76 78 85 86 90 92 96 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 2017 10810 1998 2006 1055 2453 2071 5438 2969 1011 2210 4095 2028 1011 1011 1010 2090 0 0 0\n",
            "I0830 06:43:45.345890 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 2017 10810 1998 2006 1055 2453 2071 5438 2969 1011 2210 4095 2028 1011 1011 1010 2090 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.346074 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.346246 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.346727 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] indeed , it was recorded in [MASK] star that a fortunate early [MASK] ##r had once picked up on the highway a solid chunk [MASK] gold quartz which the [MASK] had freed from its inc [MASK] ##ing soil , and washed into immediate and [MASK] popularity . [SEP] rainy season , [MASK] insult show habit of body , and seldom lifted their eyes to the rift ##ed [MASK] india - ink washed skies [MASK] them . \" cass \" beard [MASK] elliot early that morning , but not with a view to [MASK] . a leak in his [MASK] roof , - - quite [MASK] with his careless , imp ##rov ##ide ##nt habits , - - had rouse ##d him at 4 a [MASK] m [SEP]\n",
            "I0830 06:43:45.346973 140311880587136 create_pretraining_data.py:151] tokens: [CLS] indeed , it was recorded in [MASK] star that a fortunate early [MASK] ##r had once picked up on the highway a solid chunk [MASK] gold quartz which the [MASK] had freed from its inc [MASK] ##ing soil , and washed into immediate and [MASK] popularity . [SEP] rainy season , [MASK] insult show habit of body , and seldom lifted their eyes to the rift ##ed [MASK] india - ink washed skies [MASK] them . \" cass \" beard [MASK] elliot early that morning , but not with a view to [MASK] . a leak in his [MASK] roof , - - quite [MASK] with his careless , imp ##rov ##ide ##nt habits , - - had rouse ##d him at 4 a [MASK] m [SEP]\n",
            "INFO:tensorflow:input_ids: 101 5262 1010 2009 2001 2680 1999 103 2732 2008 1037 19590 2220 103 2099 2018 2320 3856 2039 2006 1996 3307 1037 5024 20000 103 2751 20971 2029 1996 103 2018 10650 2013 2049 4297 103 2075 5800 1010 1998 8871 2046 6234 1998 103 6217 1012 102 16373 2161 1010 103 15301 2265 10427 1997 2303 1010 1998 15839 4196 2037 2159 2000 1996 16931 2098 103 2634 1011 10710 8871 15717 103 2068 1012 1000 16220 1000 10154 103 11759 2220 2008 2851 1010 2021 2025 2007 1037 3193 2000 103 1012 1037 17271 1999 2010 103 4412 1010 1011 1011 3243 103 2007 2010 23358 1010 17727 12298 5178 3372 14243 1010 1011 1011 2018 27384 2094 2032 2012 1018 1037 103 1049 102\n",
            "I0830 06:43:45.347201 140311880587136 create_pretraining_data.py:161] input_ids: 101 5262 1010 2009 2001 2680 1999 103 2732 2008 1037 19590 2220 103 2099 2018 2320 3856 2039 2006 1996 3307 1037 5024 20000 103 2751 20971 2029 1996 103 2018 10650 2013 2049 4297 103 2075 5800 1010 1998 8871 2046 6234 1998 103 6217 1012 102 16373 2161 1010 103 15301 2265 10427 1997 2303 1010 1998 15839 4196 2037 2159 2000 1996 16931 2098 103 2634 1011 10710 8871 15717 103 2068 1012 1000 16220 1000 10154 103 11759 2220 2008 2851 1010 2021 2025 2007 1037 3193 2000 103 1012 1037 17271 1999 2010 103 4412 1010 1011 1011 3243 103 2007 2010 23358 1010 17727 12298 5178 3372 14243 1010 1011 1011 2018 27384 2094 2032 2012 1018 1037 103 1049 102\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.347456 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "I0830 06:43:45.347663 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            "INFO:tensorflow:masked_lm_positions: 7 12 13 25 30 36 45 52 53 54 68 74 81 82 93 99 103 105 125 0\n",
            "I0830 06:43:45.347828 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 7 12 13 25 30 36 45 52 53 54 68 74 81 82 93 99 103 105 125 0\n",
            "INFO:tensorflow:masked_lm_ids: 17162 2220 4125 1997 4542 29440 20332 4233 1037 16465 2030 2682 2018 13763 5456 6644 1011 8335 1012 0\n",
            "I0830 06:43:45.348074 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 17162 2220 4125 1997 4542 29440 20332 4233 1037 16465 2030 2682 2018 13763 5456 6644 1011 8335 1012 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "I0830 06:43:45.348223 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 0\n",
            "I0830 06:43:45.348362 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 0\n",
            "INFO:tensorflow:*** Example ***\n",
            "I0830 06:43:45.348832 140311880587136 create_pretraining_data.py:149] *** Example ***\n",
            "INFO:tensorflow:tokens: [CLS] and there burst on phil ##am ##mon ' s astonished eyes a vast semi ##ci ##rcle of blue sea [MASK] ring ##ed with palaces and towers [MASK] [SEP] like most of [MASK] fellow gold - seekers , cass was super ##sti [MASK] . [SEP]\n",
            "I0830 06:43:45.349017 140311880587136 create_pretraining_data.py:151] tokens: [CLS] and there burst on phil ##am ##mon ' s astonished eyes a vast semi ##ci ##rcle of blue sea [MASK] ring ##ed with palaces and towers [MASK] [SEP] like most of [MASK] fellow gold - seekers , cass was super ##sti [MASK] . [SEP]\n",
            "INFO:tensorflow:input_ids: 101 1998 2045 6532 2006 6316 3286 8202 1005 1055 22741 2159 1037 6565 4100 6895 21769 1997 2630 2712 103 3614 2098 2007 22763 1998 7626 103 102 2066 2087 1997 103 3507 2751 1011 24071 1010 16220 2001 3565 16643 103 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.349264 140311880587136 create_pretraining_data.py:161] input_ids: 101 1998 2045 6532 2006 6316 3286 8202 1005 1055 22741 2159 1037 6565 4100 6895 21769 1997 2630 2712 103 3614 2098 2007 22763 1998 7626 103 102 2066 2087 1997 103 3507 2751 1011 24071 1010 16220 2001 3565 16643 103 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.349485 140311880587136 create_pretraining_data.py:161] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.349700 140311880587136 create_pretraining_data.py:161] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_positions: 10 20 23 27 32 39 42 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.349865 140311880587136 create_pretraining_data.py:161] masked_lm_positions: 10 20 23 27 32 39 42 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_ids: 22741 1010 2007 1012 2010 2001 20771 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "I0830 06:43:45.350054 140311880587136 create_pretraining_data.py:161] masked_lm_ids: 22741 1010 2007 1012 2010 2001 20771 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "I0830 06:43:45.350233 140311880587136 create_pretraining_data.py:161] masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
            "INFO:tensorflow:next_sentence_labels: 1\n",
            "I0830 06:43:45.350385 140311880587136 create_pretraining_data.py:161] next_sentence_labels: 1\n",
            "INFO:tensorflow:Wrote 60 total instances\n",
            "I0830 06:43:45.362751 140311880587136 create_pretraining_data.py:166] Wrote 60 total instances\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mgAsEkBKyilu",
        "outputId": "140e49c7-e194-4b52-8fee-82f3f3b40d02"
      },
      "source": [
        "!python run_pretraining.py \\\n",
        "  --input_file=tf_examples.tfrecord \\\n",
        "  --output_dir=pretraining_output \\\n",
        "  --do_train=True \\\n",
        "  --do_eval=True \\\n",
        "  --bert_config_file=/content/bert_config.json \\\n",
        "  --train_batch_size=32 \\\n",
        "  --max_seq_length=128 \\\n",
        "  --max_predictions_per_seq=20 \\\n",
        "  --num_train_steps=20 \\\n",
        "  --num_warmup_steps=10 \\\n",
        "  --learning_rate=2e-5"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /content/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:493: The name tf.app.run is deprecated. Please use tf.compat.v1.app.run instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:407: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
            "\n",
            "W0830 06:45:38.612460 139928500971392 module_wrapper.py:139] From run_pretraining.py:407: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:407: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
            "\n",
            "W0830 06:45:38.612706 139928500971392 module_wrapper.py:139] From run_pretraining.py:407: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n",
            "W0830 06:45:38.613009 139928500971392 module_wrapper.py:139] From /content/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:414: The name tf.gfile.MakeDirs is deprecated. Please use tf.io.gfile.makedirs instead.\n",
            "\n",
            "W0830 06:45:38.613770 139928500971392 module_wrapper.py:139] From run_pretraining.py:414: The name tf.gfile.MakeDirs is deprecated. Please use tf.io.gfile.makedirs instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:418: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
            "\n",
            "W0830 06:45:38.614097 139928500971392 module_wrapper.py:139] From run_pretraining.py:418: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:420: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "W0830 06:45:38.615375 139928500971392 module_wrapper.py:139] From run_pretraining.py:420: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "INFO:tensorflow:*** Input Files ***\n",
            "I0830 06:45:38.615533 139928500971392 run_pretraining.py:420] *** Input Files ***\n",
            "INFO:tensorflow:  ./tf_examples.tfrecord\n",
            "I0830 06:45:38.615684 139928500971392 run_pretraining.py:422]   ./tf_examples.tfrecord\n",
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "W0830 06:45:38.615863 139928500971392 lazy_loader.py:50] \n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "I0830 06:45:39.664522 139928500971392 utils.py:157] NumExpr defaulting to 2 threads.\n",
            "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f43394a4680>) includes params argument, but params are not passed to Estimator.\n",
            "W0830 06:45:40.017021 139928500971392 estimator.py:1994] Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f43394a4680>) includes params argument, but params are not passed to Estimator.\n",
            "INFO:tensorflow:Using config: {'_model_dir': 'pretraining_output', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            "graph_options {\n",
            "  rewrite_options {\n",
            "    meta_optimizer_iterations: ONE\n",
            "  }\n",
            "}\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f432c8be2d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': None}\n",
            "I0830 06:45:40.017895 139928500971392 estimator.py:212] Using config: {'_model_dir': 'pretraining_output', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            "graph_options {\n",
            "  rewrite_options {\n",
            "    meta_optimizer_iterations: ONE\n",
            "  }\n",
            "}\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f432c8be2d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': None}\n",
            "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
            "I0830 06:45:40.018247 139928500971392 tpu_context.py:220] _TPUContext: eval_on_tpu True\n",
            "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
            "W0830 06:45:40.018442 139928500971392 tpu_context.py:222] eval_on_tpu ignored because use_tpu is False.\n",
            "INFO:tensorflow:***** Running training *****\n",
            "I0830 06:45:40.018597 139928500971392 run_pretraining.py:459] ***** Running training *****\n",
            "INFO:tensorflow:  Batch size = 32\n",
            "I0830 06:45:40.018745 139928500971392 run_pretraining.py:460]   Batch size = 32\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "W0830 06:45:40.024546 139928500971392 deprecation.py:506] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
            "W0830 06:45:40.025010 139928500971392 deprecation.py:323] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
            "WARNING:tensorflow:From run_pretraining.py:337: The name tf.FixedLenFeature is deprecated. Please use tf.io.FixedLenFeature instead.\n",
            "\n",
            "W0830 06:45:40.033579 139928500971392 module_wrapper.py:139] From run_pretraining.py:337: The name tf.FixedLenFeature is deprecated. Please use tf.io.FixedLenFeature instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:368: parallel_interleave (from tensorflow.contrib.data.python.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.experimental.parallel_interleave(...)`.\n",
            "W0830 06:45:40.039971 139928500971392 deprecation.py:323] From run_pretraining.py:368: parallel_interleave (from tensorflow.contrib.data.python.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.experimental.parallel_interleave(...)`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/contrib/data/python/ops/interleave_ops.py:77: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_determinstic`.\n",
            "W0830 06:45:40.040194 139928500971392 deprecation.py:323] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/contrib/data/python/ops/interleave_ops.py:77: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_determinstic`.\n",
            "WARNING:tensorflow:From run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.experimental.map_and_batch(...)`.\n",
            "W0830 06:45:40.061049 139928500971392 deprecation.py:323] From run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.experimental.map_and_batch(...)`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/contrib/data/python/ops/batching.py:276: map_and_batch (from tensorflow.python.data.experimental.ops.batching) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.map(map_func, num_parallel_calls)` followed by `tf.data.Dataset.batch(batch_size, drop_remainder)`. Static tf.data optimizations will take care of using the fused implementation.\n",
            "W0830 06:45:40.061258 139928500971392 deprecation.py:323] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/contrib/data/python/ops/batching.py:276: map_and_batch (from tensorflow.python.data.experimental.ops.batching) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.map(map_func, num_parallel_calls)` followed by `tf.data.Dataset.batch(batch_size, drop_remainder)`. Static tf.data optimizations will take care of using the fused implementation.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.parse_single_example is deprecated. Please use tf.io.parse_single_example instead.\n",
            "\n",
            "W0830 06:45:40.125081 139928500971392 module_wrapper.py:139] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.parse_single_example is deprecated. Please use tf.io.parse_single_example instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "W0830 06:45:40.239695 139928500971392 deprecation.py:323] From run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "INFO:tensorflow:Calling model_fn.\n",
            "I0830 06:45:40.270004 139928500971392 estimator.py:1148] Calling model_fn.\n",
            "INFO:tensorflow:Running train on CPU\n",
            "I0830 06:45:40.270265 139928500971392 tpu_estimator.py:3124] Running train on CPU\n",
            "INFO:tensorflow:*** Features ***\n",
            "I0830 06:45:40.270651 139928500971392 run_pretraining.py:117] *** Features ***\n",
            "INFO:tensorflow:  name = input_ids, shape = (32, 128)\n",
            "I0830 06:45:40.270861 139928500971392 run_pretraining.py:119]   name = input_ids, shape = (32, 128)\n",
            "INFO:tensorflow:  name = input_mask, shape = (32, 128)\n",
            "I0830 06:45:40.271042 139928500971392 run_pretraining.py:119]   name = input_mask, shape = (32, 128)\n",
            "INFO:tensorflow:  name = masked_lm_ids, shape = (32, 20)\n",
            "I0830 06:45:40.271204 139928500971392 run_pretraining.py:119]   name = masked_lm_ids, shape = (32, 20)\n",
            "INFO:tensorflow:  name = masked_lm_positions, shape = (32, 20)\n",
            "I0830 06:45:40.271382 139928500971392 run_pretraining.py:119]   name = masked_lm_positions, shape = (32, 20)\n",
            "INFO:tensorflow:  name = masked_lm_weights, shape = (32, 20)\n",
            "I0830 06:45:40.271552 139928500971392 run_pretraining.py:119]   name = masked_lm_weights, shape = (32, 20)\n",
            "INFO:tensorflow:  name = next_sentence_labels, shape = (32, 1)\n",
            "I0830 06:45:40.271711 139928500971392 run_pretraining.py:119]   name = next_sentence_labels, shape = (32, 1)\n",
            "INFO:tensorflow:  name = segment_ids, shape = (32, 128)\n",
            "I0830 06:45:40.271882 139928500971392 run_pretraining.py:119]   name = segment_ids, shape = (32, 128)\n",
            "WARNING:tensorflow:From /content/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
            "\n",
            "W0830 06:45:40.272193 139928500971392 module_wrapper.py:139] From /content/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
            "\n",
            "W0830 06:45:40.273645 139928500971392 module_wrapper.py:139] From /content/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
            "\n",
            "W0830 06:45:40.295802 139928500971392 module_wrapper.py:139] From /content/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "W0830 06:45:40.330669 139928500971392 deprecation.py:506] From /content/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "WARNING:tensorflow:From /content/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.Dense instead.\n",
            "W0830 06:45:40.345689 139928500971392 deprecation.py:323] From /content/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.Dense instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.__call__` method instead.\n",
            "W0830 06:45:40.346671 139928500971392 deprecation.py:323] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.__call__` method instead.\n",
            "WARNING:tensorflow:From run_pretraining.py:150: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
            "\n",
            "W0830 06:45:40.776402 139928500971392 module_wrapper.py:139] From run_pretraining.py:150: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
            "\n",
            "INFO:tensorflow:**** Trainable Variables ****\n",
            "I0830 06:45:40.776729 139928500971392 run_pretraining.py:167] **** Trainable Variables ****\n",
            "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 128)\n",
            "I0830 06:45:40.776969 139928500971392 run_pretraining.py:173]   name = bert/embeddings/word_embeddings:0, shape = (30522, 128)\n",
            "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 128)\n",
            "I0830 06:45:40.777216 139928500971392 run_pretraining.py:173]   name = bert/embeddings/token_type_embeddings:0, shape = (2, 128)\n",
            "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 128)\n",
            "I0830 06:45:40.777407 139928500971392 run_pretraining.py:173]   name = bert/embeddings/position_embeddings:0, shape = (512, 128)\n",
            "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:40.777577 139928500971392 run_pretraining.py:173]   name = bert/embeddings/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:40.777784 139928500971392 run_pretraining.py:173]   name = bert/embeddings/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.777984 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (128,)\n",
            "I0830 06:45:40.778167 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.778342 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (128,)\n",
            "I0830 06:45:40.778509 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.778731 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (128,)\n",
            "I0830 06:45:40.778917 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.779107 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:40.779303 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:40.779474 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:40.779643 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "I0830 06:45:40.779821 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (512,)\n",
            "I0830 06:45:40.780018 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (512,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (512, 128)\n",
            "I0830 06:45:40.780201 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/dense/kernel:0, shape = (512, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:40.780379 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:40.780569 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:40.780767 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.780960 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (128,)\n",
            "I0830 06:45:40.781133 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.781304 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (128,)\n",
            "I0830 06:45:40.781479 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.781656 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (128,)\n",
            "I0830 06:45:40.781839 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.782026 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:40.782226 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:40.782392 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:40.782572 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "I0830 06:45:40.782750 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (512,)\n",
            "I0830 06:45:40.782938 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (512,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (512, 128)\n",
            "I0830 06:45:40.783111 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/dense/kernel:0, shape = (512, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:40.783285 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:40.783467 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:40.783655 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.783846 139928500971392 run_pretraining.py:173]   name = bert/pooler/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:40.784075 139928500971392 run_pretraining.py:173]   name = bert/pooler/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:40.784259 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:40.784464 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:40.784635 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:40.784811 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,)\n",
            "I0830 06:45:40.785021 139928500971392 run_pretraining.py:173]   name = cls/predictions/output_bias:0, shape = (30522,)\n",
            "INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 128)\n",
            "I0830 06:45:40.785206 139928500971392 run_pretraining.py:173]   name = cls/seq_relationship/output_weights:0, shape = (2, 128)\n",
            "INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,)\n",
            "I0830 06:45:40.785392 139928500971392 run_pretraining.py:173]   name = cls/seq_relationship/output_bias:0, shape = (2,)\n",
            "WARNING:tensorflow:From /content/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
            "\n",
            "W0830 06:45:40.785605 139928500971392 module_wrapper.py:139] From /content/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
            "\n",
            "WARNING:tensorflow:From /content/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
            "\n",
            "W0830 06:45:40.786353 139928500971392 module_wrapper.py:139] From /content/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "W0830 06:45:40.951397 139928500971392 deprecation.py:323] From /usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "INFO:tensorflow:Done calling model_fn.\n",
            "I0830 06:45:42.549092 139928500971392 estimator.py:1150] Done calling model_fn.\n",
            "INFO:tensorflow:Create CheckpointSaverHook.\n",
            "I0830 06:45:42.550571 139928500971392 basic_session_run_hooks.py:541] Create CheckpointSaverHook.\n",
            "INFO:tensorflow:Graph was finalized.\n",
            "I0830 06:45:43.207135 139928500971392 monitored_session.py:240] Graph was finalized.\n",
            "2021-08-30 06:45:43.207681: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
            "2021-08-30 06:45:43.212748: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2299995000 Hz\n",
            "2021-08-30 06:45:43.213091: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55df316e8bc0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
            "2021-08-30 06:45:43.213158: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
            "2021-08-30 06:45:43.215281: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
            "2021-08-30 06:45:43.406762: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:43.407786: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55df316e9480 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
            "2021-08-30 06:45:43.407835: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Tesla K80, Compute Capability 3.7\n",
            "2021-08-30 06:45:43.409146: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:43.409819: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: \n",
            "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
            "pciBusID: 0000:00:04.0\n",
            "2021-08-30 06:45:43.420681: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2021-08-30 06:45:43.632091: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "2021-08-30 06:45:43.724238: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
            "2021-08-30 06:45:43.752910: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
            "2021-08-30 06:45:43.994992: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
            "2021-08-30 06:45:44.132814: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
            "2021-08-30 06:45:44.658049: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
            "2021-08-30 06:45:44.658270: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:44.659140: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:44.659897: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0\n",
            "2021-08-30 06:45:44.660028: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2021-08-30 06:45:44.662164: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
            "2021-08-30 06:45:44.662208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 \n",
            "2021-08-30 06:45:44.662230: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N \n",
            "2021-08-30 06:45:44.662467: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:44.663395: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:44.664289: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
            "2021-08-30 06:45:44.664351: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10813 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)\n",
            "INFO:tensorflow:Running local_init_op.\n",
            "I0830 06:45:45.498848 139928500971392 session_manager.py:500] Running local_init_op.\n",
            "INFO:tensorflow:Done running local_init_op.\n",
            "I0830 06:45:45.534147 139928500971392 session_manager.py:502] Done running local_init_op.\n",
            "INFO:tensorflow:Saving checkpoints for 0 into pretraining_output/model.ckpt.\n",
            "I0830 06:45:47.008096 139928500971392 basic_session_run_hooks.py:606] Saving checkpoints for 0 into pretraining_output/model.ckpt.\n",
            "2021-08-30 06:45:48.099622: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "INFO:tensorflow:global_step/sec: 1.37507\n",
            "I0830 06:45:50.183640 139928500971392 tpu_estimator.py:2307] global_step/sec: 1.37507\n",
            "INFO:tensorflow:examples/sec: 44.0024\n",
            "I0830 06:45:50.184217 139928500971392 tpu_estimator.py:2308] examples/sec: 44.0024\n",
            "INFO:tensorflow:global_step/sec: 11.3991\n",
            "I0830 06:45:50.271253 139928500971392 tpu_estimator.py:2307] global_step/sec: 11.3991\n",
            "INFO:tensorflow:examples/sec: 364.771\n",
            "I0830 06:45:50.271466 139928500971392 tpu_estimator.py:2308] examples/sec: 364.771\n",
            "INFO:tensorflow:global_step/sec: 12.1642\n",
            "I0830 06:45:50.353473 139928500971392 tpu_estimator.py:2307] global_step/sec: 12.1642\n",
            "INFO:tensorflow:examples/sec: 389.254\n",
            "I0830 06:45:50.353885 139928500971392 tpu_estimator.py:2308] examples/sec: 389.254\n",
            "INFO:tensorflow:global_step/sec: 12.6389\n",
            "I0830 06:45:50.432585 139928500971392 tpu_estimator.py:2307] global_step/sec: 12.6389\n",
            "INFO:tensorflow:examples/sec: 404.444\n",
            "I0830 06:45:50.432824 139928500971392 tpu_estimator.py:2308] examples/sec: 404.444\n",
            "INFO:tensorflow:global_step/sec: 13.064\n",
            "I0830 06:45:50.509119 139928500971392 tpu_estimator.py:2307] global_step/sec: 13.064\n",
            "INFO:tensorflow:examples/sec: 418.048\n",
            "I0830 06:45:50.509323 139928500971392 tpu_estimator.py:2308] examples/sec: 418.048\n",
            "INFO:tensorflow:global_step/sec: 13.3489\n",
            "I0830 06:45:50.584067 139928500971392 tpu_estimator.py:2307] global_step/sec: 13.3489\n",
            "INFO:tensorflow:examples/sec: 427.165\n",
            "I0830 06:45:50.584262 139928500971392 tpu_estimator.py:2308] examples/sec: 427.165\n",
            "INFO:tensorflow:global_step/sec: 14.0172\n",
            "I0830 06:45:50.655402 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.0172\n",
            "INFO:tensorflow:examples/sec: 448.551\n",
            "I0830 06:45:50.655634 139928500971392 tpu_estimator.py:2308] examples/sec: 448.551\n",
            "INFO:tensorflow:global_step/sec: 13.8959\n",
            "I0830 06:45:50.727320 139928500971392 tpu_estimator.py:2307] global_step/sec: 13.8959\n",
            "INFO:tensorflow:examples/sec: 444.67\n",
            "I0830 06:45:50.727500 139928500971392 tpu_estimator.py:2308] examples/sec: 444.67\n",
            "INFO:tensorflow:global_step/sec: 14.4453\n",
            "I0830 06:45:50.796549 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.4453\n",
            "INFO:tensorflow:examples/sec: 462.249\n",
            "I0830 06:45:50.796801 139928500971392 tpu_estimator.py:2308] examples/sec: 462.249\n",
            "INFO:tensorflow:global_step/sec: 14.665\n",
            "I0830 06:45:50.864795 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.665\n",
            "INFO:tensorflow:examples/sec: 469.28\n",
            "I0830 06:45:50.865049 139928500971392 tpu_estimator.py:2308] examples/sec: 469.28\n",
            "INFO:tensorflow:global_step/sec: 14.6154\n",
            "I0830 06:45:50.933178 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.6154\n",
            "INFO:tensorflow:examples/sec: 467.692\n",
            "I0830 06:45:50.933368 139928500971392 tpu_estimator.py:2308] examples/sec: 467.692\n",
            "INFO:tensorflow:global_step/sec: 14.6044\n",
            "I0830 06:45:51.001648 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.6044\n",
            "INFO:tensorflow:examples/sec: 467.34\n",
            "I0830 06:45:51.001868 139928500971392 tpu_estimator.py:2308] examples/sec: 467.34\n",
            "INFO:tensorflow:global_step/sec: 14.8045\n",
            "I0830 06:45:51.069187 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.8045\n",
            "INFO:tensorflow:examples/sec: 473.745\n",
            "I0830 06:45:51.069384 139928500971392 tpu_estimator.py:2308] examples/sec: 473.745\n",
            "INFO:tensorflow:global_step/sec: 14.8363\n",
            "I0830 06:45:51.136619 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.8363\n",
            "INFO:tensorflow:examples/sec: 474.762\n",
            "I0830 06:45:51.136837 139928500971392 tpu_estimator.py:2308] examples/sec: 474.762\n",
            "INFO:tensorflow:global_step/sec: 14.7889\n",
            "I0830 06:45:51.204262 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.7889\n",
            "INFO:tensorflow:examples/sec: 473.246\n",
            "I0830 06:45:51.204678 139928500971392 tpu_estimator.py:2308] examples/sec: 473.246\n",
            "INFO:tensorflow:global_step/sec: 14.5865\n",
            "I0830 06:45:51.272830 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.5865\n",
            "INFO:tensorflow:examples/sec: 466.768\n",
            "I0830 06:45:51.273249 139928500971392 tpu_estimator.py:2308] examples/sec: 466.768\n",
            "INFO:tensorflow:global_step/sec: 14.5959\n",
            "I0830 06:45:51.341365 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.5959\n",
            "INFO:tensorflow:examples/sec: 467.07\n",
            "I0830 06:45:51.341670 139928500971392 tpu_estimator.py:2308] examples/sec: 467.07\n",
            "INFO:tensorflow:global_step/sec: 14.4129\n",
            "I0830 06:45:51.410684 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.4129\n",
            "INFO:tensorflow:examples/sec: 461.213\n",
            "I0830 06:45:51.411044 139928500971392 tpu_estimator.py:2308] examples/sec: 461.213\n",
            "INFO:tensorflow:global_step/sec: 14.5338\n",
            "I0830 06:45:51.479486 139928500971392 tpu_estimator.py:2307] global_step/sec: 14.5338\n",
            "INFO:tensorflow:examples/sec: 465.081\n",
            "I0830 06:45:51.479718 139928500971392 tpu_estimator.py:2308] examples/sec: 465.081\n",
            "INFO:tensorflow:Saving checkpoints for 20 into pretraining_output/model.ckpt.\n",
            "I0830 06:45:51.480424 139928500971392 basic_session_run_hooks.py:606] Saving checkpoints for 20 into pretraining_output/model.ckpt.\n",
            "INFO:tensorflow:Loss for final step: 10.8629265.\n",
            "I0830 06:45:51.782692 139928500971392 estimator.py:371] Loss for final step: 10.8629265.\n",
            "INFO:tensorflow:training_loop marked as finished\n",
            "I0830 06:45:51.783103 139928500971392 error_handling.py:101] training_loop marked as finished\n",
            "INFO:tensorflow:***** Running evaluation *****\n",
            "I0830 06:45:51.783284 139928500971392 run_pretraining.py:469] ***** Running evaluation *****\n",
            "INFO:tensorflow:  Batch size = 8\n",
            "I0830 06:45:51.783416 139928500971392 run_pretraining.py:470]   Batch size = 8\n",
            "INFO:tensorflow:Calling model_fn.\n",
            "I0830 06:45:51.822953 139928500971392 estimator.py:1148] Calling model_fn.\n",
            "INFO:tensorflow:Running eval on CPU\n",
            "I0830 06:45:51.823169 139928500971392 tpu_estimator.py:3124] Running eval on CPU\n",
            "INFO:tensorflow:*** Features ***\n",
            "I0830 06:45:51.823520 139928500971392 run_pretraining.py:117] *** Features ***\n",
            "INFO:tensorflow:  name = input_ids, shape = (8, 128)\n",
            "I0830 06:45:51.823734 139928500971392 run_pretraining.py:119]   name = input_ids, shape = (8, 128)\n",
            "INFO:tensorflow:  name = input_mask, shape = (8, 128)\n",
            "I0830 06:45:51.823926 139928500971392 run_pretraining.py:119]   name = input_mask, shape = (8, 128)\n",
            "INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n",
            "I0830 06:45:51.824101 139928500971392 run_pretraining.py:119]   name = masked_lm_ids, shape = (8, 20)\n",
            "INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n",
            "I0830 06:45:51.824290 139928500971392 run_pretraining.py:119]   name = masked_lm_positions, shape = (8, 20)\n",
            "INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n",
            "I0830 06:45:51.824436 139928500971392 run_pretraining.py:119]   name = masked_lm_weights, shape = (8, 20)\n",
            "INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n",
            "I0830 06:45:51.824599 139928500971392 run_pretraining.py:119]   name = next_sentence_labels, shape = (8, 1)\n",
            "INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n",
            "I0830 06:45:51.824763 139928500971392 run_pretraining.py:119]   name = segment_ids, shape = (8, 128)\n",
            "INFO:tensorflow:**** Trainable Variables ****\n",
            "I0830 06:45:52.258430 139928500971392 run_pretraining.py:167] **** Trainable Variables ****\n",
            "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 128)\n",
            "I0830 06:45:52.258744 139928500971392 run_pretraining.py:173]   name = bert/embeddings/word_embeddings:0, shape = (30522, 128)\n",
            "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 128)\n",
            "I0830 06:45:52.259006 139928500971392 run_pretraining.py:173]   name = bert/embeddings/token_type_embeddings:0, shape = (2, 128)\n",
            "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 128)\n",
            "I0830 06:45:52.259271 139928500971392 run_pretraining.py:173]   name = bert/embeddings/position_embeddings:0, shape = (512, 128)\n",
            "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:52.259562 139928500971392 run_pretraining.py:173]   name = bert/embeddings/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:52.259840 139928500971392 run_pretraining.py:173]   name = bert/embeddings/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.260065 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (128,)\n",
            "I0830 06:45:52.260316 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.260525 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (128,)\n",
            "I0830 06:45:52.260812 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.261101 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (128,)\n",
            "I0830 06:45:52.261364 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.261591 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:52.261848 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:52.262103 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:52.262318 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "I0830 06:45:52.262520 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (512,)\n",
            "I0830 06:45:52.262749 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (512,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (512, 128)\n",
            "I0830 06:45:52.262969 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/dense/kernel:0, shape = (512, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:52.263208 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:52.263411 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:52.263644 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.263865 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (128,)\n",
            "I0830 06:45:52.264141 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.264347 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (128,)\n",
            "I0830 06:45:52.264554 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.264780 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (128,)\n",
            "I0830 06:45:52.265008 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.265248 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:52.265465 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:52.265695 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:52.265893 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "I0830 06:45:52.266135 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (128, 512)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (512,)\n",
            "I0830 06:45:52.266348 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (512,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (512, 128)\n",
            "I0830 06:45:52.266556 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/dense/kernel:0, shape = (512, 128)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:52.266777 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:52.267050 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:52.267261 139928500971392 run_pretraining.py:173]   name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.267421 139928500971392 run_pretraining.py:173]   name = bert/pooler/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:52.267604 139928500971392 run_pretraining.py:173]   name = bert/pooler/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (128, 128)\n",
            "I0830 06:45:52.267786 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/dense/kernel:0, shape = (128, 128)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (128,)\n",
            "I0830 06:45:52.267981 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/dense/bias:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (128,)\n",
            "I0830 06:45:52.268160 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/LayerNorm/beta:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (128,)\n",
            "I0830 06:45:52.268335 139928500971392 run_pretraining.py:173]   name = cls/predictions/transform/LayerNorm/gamma:0, shape = (128,)\n",
            "INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (30522,)\n",
            "I0830 06:45:52.268494 139928500971392 run_pretraining.py:173]   name = cls/predictions/output_bias:0, shape = (30522,)\n",
            "INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 128)\n",
            "I0830 06:45:52.268671 139928500971392 run_pretraining.py:173]   name = cls/seq_relationship/output_weights:0, shape = (2, 128)\n",
            "INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,)\n",
            "I0830 06:45:52.268872 139928500971392 run_pretraining.py:173]   name = cls/seq_relationship/output_bias:0, shape = (2,)\n",
            "WARNING:tensorflow:From run_pretraining.py:198: The name tf.metrics.accuracy is deprecated. Please use tf.compat.v1.metrics.accuracy instead.\n",
            "\n",
            "W0830 06:45:52.278588 139928500971392 module_wrapper.py:139] From run_pretraining.py:198: The name tf.metrics.accuracy is deprecated. Please use tf.compat.v1.metrics.accuracy instead.\n",
            "\n",
            "WARNING:tensorflow:From run_pretraining.py:202: The name tf.metrics.mean is deprecated. Please use tf.compat.v1.metrics.mean instead.\n",
            "\n",
            "W0830 06:45:52.304708 139928500971392 module_wrapper.py:139] From run_pretraining.py:202: The name tf.metrics.mean is deprecated. Please use tf.compat.v1.metrics.mean instead.\n",
            "\n",
            "INFO:tensorflow:Done calling model_fn.\n",
            "I0830 06:45:52.386525 139928500971392 estimator.py:1150] Done calling model_fn.\n",
            "INFO:tensorflow:Starting evaluation at 2021-08-30T06:45:52Z\n",
            "I0830 06:45:52.407118 139928500971392 evaluation.py:255] Starting evaluation at 2021-08-30T06:45:52Z\n",
            "INFO:tensorflow:Graph was finalized.\n",
            "I0830 06:45:52.580680 139928500971392 monitored_session.py:240] Graph was finalized.\n",
            "2021-08-30 06:45:52.581398: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:52.582218: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: \n",
            "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
            "pciBusID: 0000:00:04.0\n",
            "2021-08-30 06:45:52.582309: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
            "2021-08-30 06:45:52.582349: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
            "2021-08-30 06:45:52.582392: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
            "2021-08-30 06:45:52.582430: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
            "2021-08-30 06:45:52.582468: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
            "2021-08-30 06:45:52.582509: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
            "2021-08-30 06:45:52.582551: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
            "2021-08-30 06:45:52.582650: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:52.583388: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:52.584113: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0\n",
            "2021-08-30 06:45:52.584167: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
            "2021-08-30 06:45:52.584189: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 \n",
            "2021-08-30 06:45:52.584203: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N \n",
            "2021-08-30 06:45:52.584329: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:52.585035: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2021-08-30 06:45:52.585712: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10813 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)\n",
            "INFO:tensorflow:Restoring parameters from pretraining_output/model.ckpt-20\n",
            "I0830 06:45:52.587070 139928500971392 saver.py:1284] Restoring parameters from pretraining_output/model.ckpt-20\n",
            "INFO:tensorflow:Running local_init_op.\n",
            "I0830 06:45:52.722748 139928500971392 session_manager.py:500] Running local_init_op.\n",
            "INFO:tensorflow:Done running local_init_op.\n",
            "I0830 06:45:52.746170 139928500971392 session_manager.py:502] Done running local_init_op.\n",
            "INFO:tensorflow:Evaluation [10/100]\n",
            "I0830 06:45:53.122172 139928500971392 evaluation.py:167] Evaluation [10/100]\n",
            "INFO:tensorflow:Evaluation [20/100]\n",
            "I0830 06:45:53.294339 139928500971392 evaluation.py:167] Evaluation [20/100]\n",
            "INFO:tensorflow:Evaluation [30/100]\n",
            "I0830 06:45:53.460506 139928500971392 evaluation.py:167] Evaluation [30/100]\n",
            "INFO:tensorflow:Evaluation [40/100]\n",
            "I0830 06:45:53.626928 139928500971392 evaluation.py:167] Evaluation [40/100]\n",
            "INFO:tensorflow:Evaluation [50/100]\n",
            "I0830 06:45:53.799292 139928500971392 evaluation.py:167] Evaluation [50/100]\n",
            "INFO:tensorflow:Evaluation [60/100]\n",
            "I0830 06:45:53.969908 139928500971392 evaluation.py:167] Evaluation [60/100]\n",
            "INFO:tensorflow:Evaluation [70/100]\n",
            "I0830 06:45:54.142648 139928500971392 evaluation.py:167] Evaluation [70/100]\n",
            "INFO:tensorflow:Evaluation [80/100]\n",
            "I0830 06:45:54.308279 139928500971392 evaluation.py:167] Evaluation [80/100]\n",
            "INFO:tensorflow:Evaluation [90/100]\n",
            "I0830 06:45:54.477550 139928500971392 evaluation.py:167] Evaluation [90/100]\n",
            "INFO:tensorflow:Evaluation [100/100]\n",
            "I0830 06:45:54.646434 139928500971392 evaluation.py:167] Evaluation [100/100]\n",
            "INFO:tensorflow:Finished evaluation at 2021-08-30-06:45:54\n",
            "I0830 06:45:54.680827 139928500971392 evaluation.py:275] Finished evaluation at 2021-08-30-06:45:54\n",
            "INFO:tensorflow:Saving dict for global step 20: global_step = 20, loss = 10.850022, masked_lm_accuracy = 0.0099364715, masked_lm_loss = 10.159208, next_sentence_accuracy = 0.53625, next_sentence_loss = 0.6905006\n",
            "I0830 06:45:54.681104 139928500971392 estimator.py:2049] Saving dict for global step 20: global_step = 20, loss = 10.850022, masked_lm_accuracy = 0.0099364715, masked_lm_loss = 10.159208, next_sentence_accuracy = 0.53625, next_sentence_loss = 0.6905006\n",
            "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 20: pretraining_output/model.ckpt-20\n",
            "I0830 06:45:54.824039 139928500971392 estimator.py:2109] Saving 'checkpoint_path' summary for global step 20: pretraining_output/model.ckpt-20\n",
            "INFO:tensorflow:evaluation_loop marked as finished\n",
            "I0830 06:45:54.824702 139928500971392 error_handling.py:101] evaluation_loop marked as finished\n",
            "INFO:tensorflow:***** Eval results *****\n",
            "I0830 06:45:54.824967 139928500971392 run_pretraining.py:483] ***** Eval results *****\n",
            "INFO:tensorflow:  global_step = 20\n",
            "I0830 06:45:54.825134 139928500971392 run_pretraining.py:485]   global_step = 20\n",
            "INFO:tensorflow:  loss = 10.850022\n",
            "I0830 06:45:54.825386 139928500971392 run_pretraining.py:485]   loss = 10.850022\n",
            "INFO:tensorflow:  masked_lm_accuracy = 0.0099364715\n",
            "I0830 06:45:54.825539 139928500971392 run_pretraining.py:485]   masked_lm_accuracy = 0.0099364715\n",
            "INFO:tensorflow:  masked_lm_loss = 10.159208\n",
            "I0830 06:45:54.825690 139928500971392 run_pretraining.py:485]   masked_lm_loss = 10.159208\n",
            "INFO:tensorflow:  next_sentence_accuracy = 0.53625\n",
            "I0830 06:45:54.825829 139928500971392 run_pretraining.py:485]   next_sentence_accuracy = 0.53625\n",
            "INFO:tensorflow:  next_sentence_loss = 0.6905006\n",
            "I0830 06:45:54.825996 139928500971392 run_pretraining.py:485]   next_sentence_loss = 0.6905006\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HAhrj_Jpyiny"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rv7rWzO5yirF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vqk9xQ3jyitw"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}