{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison test\n",
    "\n",
    "This test compares the output from  `run_classifier.py` in the huggingface port to `bert_sklearn` on a small test subset from sst-2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  `run_classifier.py` from huggingface port"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Pytorch checkpoint\n",
      "Loading Pytorch checkpoint\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "07/18/2019 02:11:58 - INFO - __main__ -   device: cuda n_gpu: 1, distributed training: False, 16-bits training: False\n",
      "07/18/2019 02:11:59 - INFO - bert_sklearn.model.pytorch_pretrained.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.cache/torch/pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "07/18/2019 02:12:00 - INFO - bert_sklearn.model.pytorch_pretrained.modeling -   loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /root/.cache/torch/pytorch_pretrained_bert/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n",
      "07/18/2019 02:12:00 - INFO - bert_sklearn.model.pytorch_pretrained.modeling -   loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /root/.cache/torch/pytorch_pretrained_bert/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.bf3b9ea126d8c0001ee8a1e8b92229871d06d36d8808208cc2449280da87785c\n",
      "07/18/2019 02:12:00 - INFO - bert_sklearn.model.pytorch_pretrained.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "07/18/2019 02:12:02 - INFO - bert_sklearn.model.pytorch_pretrained.modeling -   Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "07/18/2019 02:12:02 - INFO - bert_sklearn.model.pytorch_pretrained.modeling -   Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "07/18/2019 02:12:04 - INFO - __main__ -   ***** Running training *****\n",
      "07/18/2019 02:12:04 - INFO - __main__ -     Num examples = 200\n",
      "07/18/2019 02:12:04 - INFO - __main__ -     Batch size = 16\n",
      "07/18/2019 02:12:04 - INFO - __main__ -     Num steps = 26\n",
      "\r",
      "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]\n",
      "\r",
      "Iteration:   0%|          | 0/13 [00:00<?, ?it/s]\u001b[A\n",
      "\r",
      "Iteration:   8%|▊         | 1/13 [00:00<00:03,  3.19it/s]\u001b[A\n",
      "\r",
      "Iteration:  15%|█▌        | 2/13 [00:00<00:03,  3.34it/s]\u001b[A\n",
      "\r",
      "Iteration:  23%|██▎       | 3/13 [00:00<00:02,  3.48it/s]\u001b[A\n",
      "\r",
      "Iteration:  31%|███       | 4/13 [00:01<00:02,  3.58it/s]\u001b[A\n",
      "\r",
      "Iteration:  38%|███▊      | 5/13 [00:01<00:02,  3.66it/s]\u001b[A\n",
      "\r",
      "Iteration:  46%|████▌     | 6/13 [00:01<00:01,  3.71it/s]\u001b[A\n",
      "\r",
      "Iteration:  54%|█████▍    | 7/13 [00:01<00:01,  3.76it/s]\u001b[A\n",
      "\r",
      "Iteration:  62%|██████▏   | 8/13 [00:02<00:01,  3.79it/s]\u001b[A\n",
      "\r",
      "Iteration:  69%|██████▉   | 9/13 [00:02<00:01,  3.81it/s]\u001b[A\n",
      "\r",
      "Iteration:  77%|███████▋  | 10/13 [00:02<00:00,  3.81it/s]\u001b[A\n",
      "\r",
      "Iteration:  85%|████████▍ | 11/13 [00:02<00:00,  3.83it/s]\u001b[A\n",
      "\r",
      "Iteration:  92%|█████████▏| 12/13 [00:03<00:00,  3.83it/s]\u001b[A\n",
      "\r",
      "Iteration: 100%|██████████| 13/13 [00:03<00:00,  4.18it/s]\u001b[A\n",
      "\u001b[A\r",
      "Epoch:  50%|█████     | 1/2 [00:03<00:03,  3.36s/it]\n",
      "\r",
      "Iteration:   0%|          | 0/13 [00:00<?, ?it/s]\u001b[A\n",
      "\r",
      "Iteration:   8%|▊         | 1/13 [00:00<00:03,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  15%|█▌        | 2/13 [00:00<00:02,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  23%|██▎       | 3/13 [00:00<00:02,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  31%|███       | 4/13 [00:01<00:02,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  38%|███▊      | 5/13 [00:01<00:02,  3.81it/s]\u001b[A\n",
      "\r",
      "Iteration:  46%|████▌     | 6/13 [00:01<00:01,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  54%|█████▍    | 7/13 [00:01<00:01,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  62%|██████▏   | 8/13 [00:02<00:01,  3.83it/s]\u001b[A\n",
      "\r",
      "Iteration:  69%|██████▉   | 9/13 [00:02<00:01,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  77%|███████▋  | 10/13 [00:02<00:00,  3.81it/s]\u001b[A\n",
      "\r",
      "Iteration:  85%|████████▍ | 11/13 [00:02<00:00,  3.82it/s]\u001b[A\n",
      "\r",
      "Iteration:  92%|█████████▏| 12/13 [00:03<00:00,  3.81it/s]\u001b[A\n",
      "\r",
      "Iteration: 100%|██████████| 13/13 [00:03<00:00,  4.15it/s]\u001b[A\n",
      "\u001b[A\r",
      "Epoch: 100%|██████████| 2/2 [00:06<00:00,  3.35s/it]\n",
      "07/18/2019 02:12:11 - INFO - bert_sklearn.model.pytorch_pretrained.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "07/18/2019 02:12:13 - INFO - bert_sklearn.model.pytorch_pretrained.tokenization -   loading vocabulary file ./tests/comptest/vocab.txt\n",
      "07/18/2019 02:12:13 - INFO - __main__ -   ***** Running evaluation *****\n",
      "07/18/2019 02:12:13 - INFO - __main__ -     Num examples = 100\n",
      "07/18/2019 02:12:13 - INFO - __main__ -     Batch size = 8\n",
      "\r",
      "Evaluating:   0%|          | 0/13 [00:00<?, ?it/s]\r",
      "Evaluating:  31%|███       | 4/13 [00:00<00:00, 32.40it/s]\r",
      "Evaluating:  62%|██████▏   | 8/13 [00:00<00:00, 32.13it/s]\r",
      "Evaluating:  92%|█████████▏| 12/13 [00:00<00:00, 32.21it/s]\r",
      "Evaluating: 100%|██████████| 13/13 [00:00<00:00, 33.16it/s]\n",
      "07/18/2019 02:12:14 - INFO - __main__ -   ***** Eval results *****\n",
      "07/18/2019 02:12:14 - INFO - __main__ -     acc = 0.88\n",
      "07/18/2019 02:12:14 - INFO - __main__ -     eval_loss = 0.31115847940628344\n",
      "07/18/2019 02:12:14 - INFO - __main__ -     global_step = 26\n",
      "07/18/2019 02:12:14 - INFO - __main__ -     loss = 0.1801566292460148\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16 ms, sys: 4 ms, total: 20 ms\n",
      "Wall time: 16.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%bash\n",
    "cd ..\n",
    "python ./tests/run_classifier.py --task_name sst-2 \\\n",
    "                            --data_dir ./tests/data/sst2 \\\n",
    "                            --do_train  --do_eval \\\n",
    "                            --output_dir ./tests/comptest \\\n",
    "                            --bert_model bert-base-uncased \\\n",
    "                            --do_lower_case \\\n",
    "                            --learning_rate 3e-5 \\\n",
    "                            --gradient_accumulation_steps 1 \\\n",
    "                            --max_seq_length 64 \\\n",
    "                            --train_batch_size 16 \\\n",
    "                            --eval_batch_size 8 \\\n",
    "                            --num_train_epochs 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc = 0.88\r\n",
      "eval_loss = 0.31115847940628344\r\n",
      "global_step = 26\r\n",
      "loss = 0.1801566292460148\r\n"
     ]
    }
   ],
   "source": [
    "!cat comptest/eval_results.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -r comptest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  `bert_sklearn` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SST-2 train data size: 200 \n",
      "SST-2 dev data size: 100 \n",
      "Building sklearn text classifier...\n",
      "Loading bert-base-uncased model...\n",
      "Defaulting to linear classifier/regressor\n",
      "Loading Pytorch checkpoint\n",
      "train data size: 200, validation data size: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training  : 100%|██████████| 13/13 [00:03<00:00,  4.03it/s, loss=0.671]\n",
      "Training  : 100%|██████████| 13/13 [00:03<00:00,  4.00it/s, loss=0.36] \n",
      "Testing: 100%|██████████| 13/13 [00:00<00:00, 22.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Loss: 0.3112, Accuracy: 88.00%\n",
      "CPU times: user 9.88 s, sys: 3.45 s, total: 13.3 s\n",
      "Wall time: 14.5 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "import sys\n",
    "import csv\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "sys.path.append(\"../\") \n",
    "from bert_sklearn import BertClassifier\n",
    "from bert_sklearn import load_model\n",
    "\n",
    "\n",
    "def get_sst_test_data(train_file ='./data/sst2/train.tsv',\n",
    "                dev_file  = './data/sst2/dev.tsv'):\n",
    "    \n",
    "    train = pd.read_csv(train_file, sep='\\t', encoding='utf8', keep_default_na=False)\n",
    "    train.columns=['text','label']\n",
    "    print(\"SST-2 train data size: %d \"%(len(train)))\n",
    "    \n",
    "    dev = pd.read_csv(dev_file, sep='\\t', encoding='utf8', keep_default_na=False)\n",
    "    dev.columns=['text','label']\n",
    "    print(\"SST-2 dev data size: %d \"%(len(dev)))\n",
    "    label_list = np.unique(train['label'])\n",
    "\n",
    "    X_train = train['text']\n",
    "    y_train = train['label']\n",
    "    X_dev = dev['text']\n",
    "    y_dev = dev['label']\n",
    "\n",
    "    return X_train,y_train, X_dev, y_dev\n",
    "\n",
    "\n",
    "X_train,y_train, X_dev, y_dev =  get_sst_test_data()\n",
    "\n",
    "# define model\n",
    "model = BertClassifier('bert-base-uncased')\n",
    "model.validation_fraction = 0.0\n",
    "model.learning_rate = 3e-5 \n",
    "model.gradient_accumulation_steps = 1\n",
    "model.max_seq_length = 64\n",
    "model.train_batch_size = 16\n",
    "model.eval_batch_size = 8\n",
    "model.epochs = 2\n",
    "\n",
    "# fit\n",
    "model.fit(X_train,y_train)\n",
    "\n",
    "# score\n",
    "accy = model.score(X_dev,y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
