{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "aiml_df = pd.read_csv('aiml_dfs/aiml_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "aiml_df['significance'] = 0\n",
    "for i, row in aiml_df.iterrows():\n",
    "    if len(row['classification_product_code'].split(',')) > 1:\n",
    "        print(i, \"multiple:\", row['classification_product_code'])\n",
    "        #columns_df.at[i, 'significance'] = 1\n",
    "        continue\n",
    "    if row['classification_product_code'].upper() in ['QBS', 'QDQ', 'POK', 'OEB', 'PIB', 'DPS']:\n",
    "        aiml_df.at[i, 'significance'] = 2\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in aiml_df.iterrows():\n",
    "    \n",
    "    with open('FDA_texts/{}.txt'.format(row['query_id']), 'r', errors='ignore') as fp:\n",
    "        text = fp.read()\n",
    "        text = text.lower().replace('\\n', ' ').replace('\\r', ' ').replace('\\t', ' ')\n",
    "        text = re.sub('[^0-9a-zA-Z]+', ' ', text)\n",
    "        \n",
    "        keyword = 'triage and notification'\n",
    "        has_keyword = int(keyword in text)\n",
    "\n",
    "        context = 50\n",
    "        if has_keyword:\n",
    "            aiml_df.at[i, 'significance'] = 1\n",
    "#             idx = []\n",
    "#             start = 0\n",
    "#             while (keyword in text[start:] and start < len(text)):\n",
    "#                 index = text[start:].index(keyword)+start\n",
    "#                 idx.append(index)\n",
    "#                 start = index + len(keyword)\n",
    "#             for ix in idx:\n",
    "#                 print(row['query_id'], text[ix-context: ix+context])\n",
    "#         else:\n",
    "#             print(row['query_id'])\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "aiml_df['risk_level'] = 0\n",
    "for i, row in aiml_df.iterrows():\n",
    "    if row['severity'] == 2:\n",
    "        if row['significance'] == 2:\n",
    "            aiml_df.at[i, 'risk_level'] = 4\n",
    "        elif row['significance'] == 1:\n",
    "            aiml_df.at[i, 'risk_level'] = 3\n",
    "        elif row['significance'] == 0:\n",
    "            aiml_df.at[i, 'risk_level'] = 2\n",
    "    elif row['severity'] == 1:\n",
    "        if row['significance'] == 2:\n",
    "            aiml_df.at[i, 'risk_level'] = 3\n",
    "        elif row['significance'] == 1:\n",
    "            aiml_df.at[i, 'risk_level'] = 2\n",
    "        elif row['significance'] == 0:\n",
    "            aiml_df.at[i, 'risk_level'] = 1\n",
    "    elif row['severity'] == 0:\n",
    "        if row['significance'] == 2:\n",
    "            aiml_df.at[i, 'risk_level'] = 2\n",
    "        elif row['significance'] == 1:\n",
    "            aiml_df.at[i, 'risk_level'] = 1\n",
    "        elif row['significance'] == 0:\n",
    "            aiml_df.at[i, 'risk_level'] = 1\n",
    "    if aiml_df.at[i, 'risk_level'] == 0:\n",
    "        print(i, row['severity'], row['significance'])\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    65\n",
       "3    46\n",
       "1    12\n",
       "4     7\n",
       "Name: risk_level, dtype: int64"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aiml_df['risk_level'].value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
