import sys
import ctypes
import win32com.shell.shell as shell
import webbrowser
import subprocess
import time
from multiprocessing import freeze_support

from PyQt5.QtWidgets import QApplication, QWidget, QLineEdit, QVBoxLayout, QLabel, QTableView
from PyQt5.QtCore import pyqtSlot, QModelIndex, QAbstractTableModel, Qt

from DBManager import DBManager
from UIthread import *

########################################################################################################################
# UI 클래스
# UI 를 구성하기 위한 클래스
# UI 뿐만 아니라 각 UI 위젯 간의 시그널을 관리하기 위한 시그널 핸들러 함수도 정의하고 있다.
########################################################################################################################
class UI(QWidget):
    def __init__(self):
        super().__init__()
        self.db = DBManager()
        self.cached_file_list = [] # DB를 통해 검색하지 않고, 메모리에 올려서 검색을 수행하기 위한 파일 리스트
        self.count = 0 # 사용자 입력값이 2개 이상일 때 처리해주기 위해 구성

        # 테이블 갱신
        self.initUI()

        # 디렉토리를 탐색해서 파일 리스트를 얻은 후, DB에 insert 하는 작업까지 수행하는 쓰레드
        self.scan_thread = ScanThread()
        self.scan_thread.set_db(self.db)
        self.scan_thread.finish_scan_signal.connect(self.finish_scan)
        self.start_thread(mode = 1)

        # DB 정보를 읽기 위한 쓰레드
        self.read_db_thread = ReadDBThread()
        self.read_db_thread.set_db(self.db)
        self.read_db_thread.finish_read_signal.connect(self.finish_read)
        self.start_thread(mode = 0)

        # 새로 생성된 파일정보를 DB에 넣기 위한 쓰레드
        self.insert_db_thread = InsertDBThread()
        self.insert_db_thread.set_db(self.db)

        # 삭제된 파일정보를 DB에서 제거하기 위한 쓰레드
        self.delete_db_thread = DeleteDBThread()
        self.delete_db_thread.set_db(self.db)

        # 각 드라이브 별로 파일 변화 감지를 수행하는 쓰레드를 관리하는 쓰레드
        self.manager_observer_thread = ManagerObserverThread()
        self.manager_observer_thread.file_change_signal.connect(self.control_updated_file)
        self.start_thread(mode = 2)

        self.check_time = checkTime()
        self.check_time.timeout.connect(self.processing_time_out)

    ####################################################################################################################
    # initTable
    # = 테이블 표시를 위한 기본 설정을 하는 메소드이다.
    #   테이블의 entry를 클릭 했을 때, 파일이나 디렉토리를 실행하기 위한 이벤트(execute_file)를 연결해주고,
    #   테이블의 헤더를 클릭 했을 때 정렬 기능을 실행하기 위한 이벤트(sort_data)도 연결해준다.
    ###################################################################################################################
    def initTable(self):
        self.header_sorted_state = [False, False, False]
        self.table_model = TableModel()

        self.tableview.setSortingEnabled(True)

        self.tableview.doubleClicked.connect(self.execute_file)
        self.tableview.horizontalHeader().sectionClicked.connect(self.sort_data)

    ####################################################################################################################
    # inintUI
    # = UI를 초기화하기 위한 메소드이다.
    #   사용자 입력을 받기 위한 LineEdit, 검색된 결과 개수를 표시하기 위한 Label, 검색된 결과를 표시하기 위한 tableview
    #   위젯들을 포함한다.
    ####################################################################################################################
    def initUI(self):

        # 사용자 입력을 받는 위젯
        self.qle = QLineEdit(self)
        self.qle.textEdited.connect(self.update_table_data)

        # 검색된 결과의 개수를 표시하는 위젯
        self.label = QLabel(self)

        # 검색된 결과를 표시하는 테이블
        self.tableview = QTableView()
        self.initTable()

        # UI 구성
        self.vbox = QVBoxLayout()
        self.vbox.addWidget(self.qle)
        self.vbox.addWidget(self.tableview)
        self.vbox.addWidget(self.label)

        self.setLayout(self.vbox)
        self.setWindowTitle('File Search Program')
        self.setGeometry(300, 100, 1500, 800)
        self.show()

    ####################################################################################################################
    # displayFiles
    # - file_list : 테이블 상에 표시한 파일들의 정보를 담고 있는 리스트 ( 파일명, 경로, 크기를 튜플 형태로 저장)
    # = 검색 결과에 의해 갱신되는 테이블을 나타내는 메소드
    ####################################################################################################################
    def displayFiles(self, file_list):
        total = len(file_list)
        self.label.setText("총 {0} 개 검색됨".format(total))

        self.tableview.clearSpans()
        self.table_model.layoutAboutToBeChanged.emit()
        self.table_model.setData(file_list)
        self.table_model.layoutChanged.emit()

        self.tableview.setModel(self.table_model)
        self.tableview.setColumnWidth(0, 600)
        self.tableview.setColumnWidth(1, 600)
        self.tableview.setColumnWidth(2, 80)

    ########################################################################################################################
    # start_thread
    # - mode : 시작하게 될 쓰레드를 명시하기 위한 변수
    # = 인자로 받은 mode을 통해 실행할 쓰레드를 결정하고, 쓰레드를 실행한다.
    ########################################################################################################################
    def start_thread(self, mode):
        if mode == 0: #read_db_thread -> DB로부터 파일 정보 읽기
            self.read_db_thread.start()

        elif mode == 1: #scan_thread -> 파일 시스템 스캔 및 DB 저장
            self.scan_thread.start()

        elif mode == 2: #manager_observer_thread -> 파일 변화를 감지하는 Observer를 thread을 생성하고, 이벤트 처리
            self.manager_observer_thread.start()

        elif mode == 3: #insert_db_thread -> DB에 추가된 파일 정보 저장
            self.insert_db_thread.start()

        elif mode == 4: #delete_db_thread -> DB에서 삭제된 파일 정보 제거
            self.delete_db_thread.start()

    ########################################################################################################################
    # insert_fileinfo
    # - file_info : 추가하게 될 파일 정보가 들어있는 리스트(파일명, 경로, 사이즈)
    # = 인자로 받은 file_info를 통해 파일 정보를 갱신하고, 테이블을 갱신하는 메소드를 호출한다.
    ########################################################################################################################
    def insert_fileinfo(self, file_info):
        self.cached_file_list.append(file_info)
        self.check_time.start()

    ########################################################################################################################
    # delete_fileinfo
    # - file_info : 삭제하게 될 파일 정보가 들어있는 리스트(파일명, 경로)
    # = 인자로 받은 file_info를 통해 파일명과 경로명이 일치하는 파일 정보를 삭제하고, 테이블을 갱신하는 메소드를 호출한다.
    ########################################################################################################################
    def delete_fileinfo(self, file_info):
        file_name = file_info[0]
        dir_name = file_info[1]

        for f in self.cached_file_list:
            if f[0] == file_name and f[1] == dir_name:
                self.cached_file_list.remove(f)
                break

        self.check_time.start()

    def processing_time_out(self):
        self.update_table_data(True)

########################################################################################################################
#     이벤트 핸들러 메소드
########################################################################################################################

    ####################################################################################################################
    # sort_data
    # - column_index : 선택 된 헤더의 column_index
    # = 헤더(File, Path, Size(kb))를 클릭했을 때 정렬을 수행하기 위한 이벤트 처리 메소드
    #   각 헤더 마다 가지고 있는 header_sorted_state 정보를 통해 해당 열에 대한 정렬 기준을 수립한다.
    #   False -> 오름 차순으로, True -> 내림 차순으로 적용
    ####################################################################################################################
    @pyqtSlot(int)
    def sort_data(self, column_index):
        self.table_model.layoutAboutToBeChanged.emit()
        table_data = self.table_model.getData()

        if self.header_sorted_state[column_index]: # 내림 차순
            table_data = sorted(table_data, key=lambda f: f[column_index] if column_index==2 else f[column_index].upper(), reverse = True)
            self.header_sorted_state[column_index] = False
        else: # 오름 차순
            table_data = sorted(table_data, key = lambda f: f[column_index] if column_index==2 else f[column_index].upper())
            self.header_sorted_state[column_index] = True

        self.table_model.setData(table_data)
        self.table_model.layoutChanged.emit()

    ####################################################################################################################
    # execute_file
    # - signal : 선택된 tableview의 entry에 대한 정보를 담고 있는 QModeIndex 객체
    # = 테이블 상에서 entry을 더블 클릭했을 때 파일 실행 또는 디렉토리 열기를 수행하기 위한 이벤트 처리 메소드
    #   경로를 더블 클릭 시 -> 해당 디렉토리 열기
    #   파일명 또는 사이즈를 더블 클릭 시 -> 해당 파일 실행
    ####################################################################################################################
    @pyqtSlot(QModelIndex)
    def execute_file(self, signal):
        row = signal.row()
        col = signal.column()

        if col == 1: # 파일 경로 더블 클릭 -> 해당 디렉토리 열기
            index = signal.sibling(row, col)
            index_dict = self.table_model.itemData(index)
            path = index_dict.get(0)
        else: # 파일 명 더블 클릭 -> 해당 파일 실행
            findex = signal.sibling(row, 0)
            findex_dict = self.table_model.itemData(findex)
            file_name = findex_dict.get(0)

            pindex = signal.sibling(row, 1)
            pindex_dict = self.table_model.itemData(pindex)
            path = pindex_dict.get(0)
            path = path + "\\" + file_name

        webbrowser.open(path)

    ####################################################################################################################
    # finish_scan
    # = scan_thread의 동작이 완료되었을 때을 위한 이벤트 처리 메소드
    #   start_thread 메소드를 통해 파일 변화를 감지하는 manager_observer_thread를 실행한다.
    #   또한 파일 시스템 스캔을 통해 얻은 파일 정보를 cached_file_list에 갱신한다.
    #   사용자 입력값에 따라 변경사항을 테이블에 표시한다.
    ####################################################################################################################
    @pyqtSlot(list)
    def finish_scan(self, file_list):
        self.finish_scan_flag = True
        self.cached_file_list = file_list
        len(self.cached_file_list)
        self.update_table_data()

    ########################################################################################################################
    # finish_read
    # = read_db_thread 에 의해 프로그램 시작 후, DB에서 파일 정보를 읽어왔을 때 시그널을 받게 되는 메소드
    #   수집된 파일 정보를 나타내는 cached_file_list를 갱신하고, 해당 값을 테이블에 표시한다.
    ########################################################################################################################
    @pyqtSlot(list)
    def finish_read(self, file_list):
        self.cached_file_list = file_list
        self.displayFiles(file_list)

    ########################################################################################################################
    # update_table_data
    # = 사용자 입력값 변경 시 또는 파일 정보에 변화가 생겼을 때, 입력값을 기준으로 테이블에 표시할 파일 정보를 생성하는 메소드
    #   생성된 파일 정보를 displayFiles 메소드를 통해 테이블 나타낸다.
    ########################################################################################################################
    @pyqtSlot()
    def update_table_data(self, time_out = False):
        text = self.qle.text().upper()

        if self.count < 2 and len(text) > 0 and not time_out:
            self.count = self.count + 1
            self.check_time.start()
            return

        self.count = 0

        # file_list의 첫번째 항목인 file_name을 토대로 text 값을 포함하고 있으면 displayed_file_list에 추가한다.
        displayed_file_list = [file_info for file_info in self.cached_file_list if text in file_info[0].upper()]
        self.displayFiles(displayed_file_list)

    ########################################################################################################################
    # control_updated_file
    # - file_infos : 파일 변경 감지에 따라 대상이 되는 파일 정보가 저장되어있는 리스트이며
    #   0번 인덱스는 실질적인 파일 정보가 들어가 있고,
    #   1번 인덱스에는 어떠한 파일 변경 이벤트가 발생했는지를 나타낸다.
    # = file_infos를 통해 파일 변경 이벤트를 구분하고, 각 이벤트에 맞게 테이블 갱신 및 DB 갱신을 수행하는 메소드를 호출한다.
    ########################################################################################################################
    @pyqtSlot(list)
    def control_updated_file(self, file_infos):
        mode = file_infos[1]
        file_info = file_infos[0]

        if mode == 3: # insert
            self.insert_fileinfo(file_info)
            self.insert_db_thread.set_file_info(file_info)

        elif mode == 4: # delete
            self.delete_fileinfo(file_info)
            self.delete_db_thread.set_file_info(file_info)

        self.start_thread(mode)

########################################################################################################################
# TableModel 클래스
# QAbstractTableModel 인터페이스를 상속받아 세부 메소드를 구현하고 있다.
# View - Model 관점에서 View는 단지 데이터를 나타내기 위한 객체이며, Model은 데이터 처리 또는 정의를 나타내는 객체이다.
# 이러한 Model의 객체에 대한 정의를 내리는 클래스이다.
########################################################################################################################
class TableModel(QAbstractTableModel):
    header_labels = ['Name', 'Path', 'Size(KB)']

    def __init__(self):
        super(TableModel, self).__init__()

    def setData(self, filelist):
        self._data = filelist

    def getData(self):
        return self._data

    def data(self, index, role):
        if role == Qt.DisplayRole:
            return self._data[index.row()][index.column()]

    def headerData(self, section, orientation, role = Qt.DisplayRole):
        if role == Qt.DisplayRole and orientation == Qt.Horizontal:
            return self.header_labels[section]
        return QAbstractTableModel.headerData(self, section, orientation, role)

    def rowCount(self, index):
        return len(self._data)

    def columnCount(self, index):
        return 3

########################################################################################################################
# is_admin
# = 프로그램이 관리자 권한으로 실행되었는지 확인하기 위한 함수
########################################################################################################################
def is_admin():
    try:
        return ctypes.windll.shell32.IsUserAnAdmin()
    except:
        return False

if __name__ == '__main__':
    freeze_support()

    # 실행 파일을 권리자 권한으로 수행하기 위해 사용하는 코드
    """
    if not is_admin():
        script = os.path.abspath(sys.argv[0])
        params = ' '.join([script] + sys.argv[1:])

        shell.ShellExecuteEx(nShow = 1, lpVerb = 'runas', lpFile = sys.executable, lpParameters=params)
        sys.exit()
    """

    try:
        app = QApplication(sys.argv)
        ex = UI()
        sys.exit(app.exec_())
    finally:
        pid = os.getpid()
        spid = str(pid)

        subprocess.run(args = ['taskkill', '/F', '/T', '/PID', spid], shell = False)


#Given an integer array arr of size n, you need to sum the elements of arr.

# Input:
# n = 3
# arr[] = {3 2 1}
# Output: 6

import numpy as np
n = 4
arr = np.array = ([3, 2, 5, 5])
sum_arr = 0
for i in arr:
    sum_arr =+ sum_arr+i

print('Soma dos elementos do array: ', sum_arr)


import shutil
import tempfile
import random
from http import HTTPStatus

from django import forms
from django.conf import settings
from django.test import Client, TestCase, override_settings
from django.urls import reverse
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.cache import cache

from yatube.constants import POSTS_PER_STR

from ..models import Group, Post, User, Comment, Follow

TEMP_MEDIA_ROOT = tempfile.mkdtemp(dir=settings.BASE_DIR)

@override_settings(MEDIA_ROOT=TEMP_MEDIA_ROOT)
class PostMainViewTests(TestCase):
    """Тесты проверки где достаточно проверить один пост."""
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.guest_client = Client()
        cls.user = User.objects.create_user(username='TestUser')
        cls.authorized_client = Client()
        cls.authorized_client.force_login(cls.user)
        cls.form_fields = {
            'text': forms.fields.CharField,
            'group': forms.fields.ChoiceField,
        }

        cls.group = Group.objects.create(
            title='Тестовая группа',
            description='Тестовое описание',
            slug='group-slug',
        )

        cls.author = User.objects.create_user(
            first_name='Лев',
            last_name='Толстой',
            username='FirstAuthor',
            email='lev@yatube.ru'
        )

        cls.author_client = Client()
        cls.author_client.force_login(PostMainViewTests.author)

        small_gif = (
            b'\x47\x49\x46\x38\x39\x61\x02\x00'
            b'\x01\x00\x80\x00\x00\x00\x00\x00'
            b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
            b'\x00\x00\x00\x2C\x00\x00\x00\x00'
            b'\x02\x00\x01\x00\x00\x02\x02\x0C'
            b'\x0A\x00\x3B'
        )

        uploaded = SimpleUploadedFile(
            name='small.gif',
            content=small_gif,
            content_type='image/gif'
        )

        cls.post = Post.objects.create(
            group=PostMainViewTests.group,
            text='Тестовый пост',
            author=cls.author,
            image=uploaded,
        )

    def setUp(self):
        self.guest_client = Client()
        self.user = User.objects.create_user(username='HasNoName')
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)
        self.author_client = Client()
        self.author_client.force_login(PostMainViewTests.author)
        self.form_fields = {
            'text': forms.fields.CharField,
            'group': forms.fields.ChoiceField,
        }

    def test_page_has_correct_template(self):
        """URL-адрес использует соответствующий шаблон."""
        templates_pages_names = {
            reverse('posts:index'): 'posts/index.html',
            reverse('posts:group', kwargs={'slug': f'{self.group.slug}'}):
                'posts/group_list.html',
            reverse('posts:profile', kwargs={'username':
                    f'{self.user.username}'}): 'posts/profile.html',
            reverse('posts:post_detail', kwargs={'post_id':
                    f'{self.post.id}'}): 'posts/post_detail.html',
            reverse('posts:post_create'): 'posts/create_post.html',
            reverse('posts:post_detail', kwargs={'post_id':
                    f'{self.post.id}'}): 'posts/post_detail.html',
        }
        for template, reverse_name in templates_pages_names.items():
            with self.subTest(reverse_name=reverse_name):
                response = self.authorized_client.get(template)
                self.assertTemplateUsed(response, reverse_name)

    def test_home_page_show_correct_context(self):
        """Пост отображается на главной странице"""
        response = self.authorized_client.get(reverse('posts:index'))
        first_object = response.context['page_obj'][0]

        post_text_0 = first_object.text
        post_group_0 = first_object.group.title
        post_author_0 = first_object.author.username
        post_image_0 = first_object.image
        self.assertEqual(post_text_0, 'Тестовый пост')
        self.assertEqual(post_group_0, 'Тестовая группа')
        self.assertEqual(post_author_0, 'FirstAuthor')
        self.assertEqual(post_image_0, 'posts/small.gif')

    def test_group_page_show_correct_context(self):
        """Проверка отображения на странице группы"""
        response = self.authorized_client.get(reverse('posts:group',
                                                      args=[self.group.slug]))
        first_object = response.context['page_obj'][0]
        post_image_0 = first_object.image
        self.assertEqual(response.status_code, HTTPStatus.OK)
        post_text_0 = first_object.text
        post_group_0 = first_object.group.title
        post_author_0 = first_object.author.username
        self.assertEqual(post_image_0, 'posts/small.gif')

    def test_post_on_author_page(self):
        """Проверка отображения нового поста на странице автора"""
        response = (self.authorized_client.
                    get(reverse('posts:profile', args=[self.author.username])))
        first_object = response.context['page_obj'][0]
        post_image_0 = first_object.image
        self.assertEqual(response.status_code, HTTPStatus.OK)
        post_text_0 = first_object.text
        post_group_0 = first_object.group.title
        post_author_0 = first_object.author.username
        self.assertEqual(post_image_0, 'posts/small.gif')

    def test_post_detail_pages_show_correct_context(self):
        """Шаблон post_detail сформирован с правильным контекстом."""
        response = (self.authorized_client.get(reverse('posts:post_detail',
                    kwargs={'post_id': f'{self.post.id}'})))
        self.assertEqual(response.context.get('post').group.title,
                         'Тестовая группа')
        self.assertEqual(response.context.get('post').text, 'Тестовый пост')
        self.assertEqual(response.context.get('post').image, 'posts/small.gif')

    def test_context_in_profile(self):
        """Шаблон post_profile сформирован с правильным контекстом."""
        response = (self.authorized_client.
            get(reverse('posts:profile', args=[self.author.username])))
        self.assertEqual(response.context.get('post').group.title,
                         'Тестовая группа')
        self.assertEqual(response.context.get('post').text, 'Тестовый пост')
        self.assertEqual(response.context.get('post').image, 'posts/small.gif')

    def test_create_post__page_show_correct_context(self):
        """Шаблон create_post сформирован с правильным контекстом."""
        response = self.authorized_client.get(reverse("posts:post_create"))

        for value, expected in self.form_fields.items():
            with self.subTest(value=value):
                form_field = response.context.get("form").fields.get(value)
                self.assertIsInstance(form_field, expected)

    def test_post_edit_page_correct_context(self):
        response = self.author_client.get(reverse('posts:post_edit',
                                          args=[PostMainViewTests.post.id]))
        for value, expected in self.form_fields.items():
            with self.subTest(value=value):
                form_field = response.context['form'].fields[value]
                self.assertIsInstance(form_field, expected)

    def test_add_comment_correct(self):
        """Комментировать посты может только авторизованный пользователь."""
        form_data = {'text': 'test_add_comment_correct', }
        add_comment_url = reverse(
            'posts:add_comment', kwargs={
                'post_id': PostMainViewTests.post.id
            }
        )
        response = self.client.post(
            add_comment_url,
            data=form_data,
            follow=True
        )
        # проверяем корректность редиректа неавторизованного пользователя
        self.assertRedirects(
            response, f'{reverse("users:login")}?next={add_comment_url}'
        )
        # создаем комментарий
        form_data = {'text': 'test_add_comment_correct', }
        response = self.authorized_client.post(
            add_comment_url,
            data=form_data,
            follow=True,
        )
        # проверяем корректность редиректа
        self.assertRedirects(
            response,
            reverse(
                'posts:post_detail', kwargs={
                    'post_id': PostMainViewTests.post.id
                }
            )
        )
        # проверяем, что первый комментарий на странице только что добавленный
        self.assertIn('comments', response.context)
        self.assertGreaterEqual(len(response.context['comments']), 1)
        self.assertEqual(
            response.context['comments'][0].text, form_data['text']
        )

class CacheIndexPageTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cache.clear()
        cls.author = User.objects.create(
            username='Author',
        )

        cls.group = Group.objects.create(
            title='Тестовая группа',
            description='Тестовое описание',
            slug='group-slug',
        )

        cls.post = Post.objects.create(
            group=cls.group,
            text='Тестовый пост для проверки кэширования.',
            author=cls.author,
        )

    def setUp(self):
        self.authorized_client = Client()
        self.authorized_client.force_login(self.author)

    def test_cach_in_index_page(self):
        """Проверяем работу кеша на главной странице"""

        # проверим, что пост есть
        response_1 = self.authorized_client.get(reverse('posts:index'))
        Post.objects.all().delete()
        response_2 = self.authorized_client.get(reverse('posts:index'))
        self.assertEqual(
            response_1.content,
            response_2.content
        )
        cache.clear()
        response_2 = self.authorized_client.get(reverse('posts:index'))
        self.assertNotEqual(
            response_1.content,
            response_2.content
        )

class PostViewTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        PAGES_COUNT_1 = 14
        cls.guest_client = Client()
        cls.user = User.objects.create_user(username='TestUser')
        cls.authorized_client = Client()
        cls.authorized_client.force_login(cls.user)

        cls.group = Group.objects.create(
            title='Тестовая группа 1',
            description='Тестовое описание 1',
            slug='group-slug',
        )

        cls.group_2 = Group.objects.create(
            title="Тестовая группа 2",
            description="Тестовое описание 2",
            slug="group_slug_2",
        )

        cls.author = User.objects.create_user(
            first_name='Лев',
            last_name='Толстой',
            username='FirstAuthor',
            email='lev@yatube.ru'
        )

        cls.author_2 = User.objects.create_user(
            first_name='Mark',
            last_name='Tven',
            username='SecondAuthor',
            email='mark@yatube.ru'
        )

        cls.post = Post.objects.bulk_create([
            Post(group=PostMainViewTests.group,
                 text=f'Тест  1-{i + 1}',
                 author=cls.author,)
            for i in range(1, PAGES_COUNT_1)
        ])

        cls.post = Post.objects.create(
            group=PostMainViewTests.group,
            text="Пост для проверки",
            author=cls.author,
        )

    def setUp(self):
        self.guest_client = Client()
        self.user = User.objects.create_user(username='HasNoName')
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)
        cache.clear()

    def test_group_page_show_correct_context(self):
        """Пост отображается на странице группы"""
        response = self.authorized_client.get(
            reverse('posts:group', args=['group-slug']))
        num = random.randint(1, POSTS_PER_STR - 1)
        first_object = response.context['page_obj'][num]
        post_group_0 = first_object.group.title
        self.assertEqual(post_group_0, 'Тестовая группа 1')

    def test_context_in_profile(self):
        """Проверка содержимого словаря context для /<username>/"""
        response = (self.authorized_client.
                    get(reverse('posts:profile', args=[self.author.username])))
        num = random.randint(1, POSTS_PER_STR - 1)
        first_object = response.context['page_obj'][num]
        post_author_0 = first_object.author.username
        self.assertEqual(post_author_0, 'FirstAuthor')

    def test_post_delete(self):
        """Проверка удаления поста"""
        Post.objects.filter(text="Тест 1-14").delete()
        response = (self.authorized_client.
                    get(reverse('posts:profile', args=[self.author.username])))
        self.assertNotContains(response, "Тест 1-14")

class paginatorViewsTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='Test User')
        cls.authorized_client = Client()
        cls.authorized_client.force_login(cls.user)
        for count in range(15):
            cls.post = Post.objects.create(
                text=f'Тестовый пост номер {count}',
                author=cls.user)

    def test_first_page_contains_ten_records(self):
        response = self.authorized_client.get(reverse('posts:index'))
        self.assertEqual(len(response.context.get('page_obj').object_list), 10)

    def test_second_page_contains_three_records(self):
        response = self.authorized_client.get(
            reverse('posts:index') + '?page=2')
        self.assertEqual(len(response.context.get('page_obj').object_list), 5)

class FollowTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.author = User.objects.create_user(username='followingUser')
        cls.author_user = Client()
        cls.author_user.force_login(cls.author)
        cls.follower = User.objects.create_user(username='followerUser')
        cls.follower_user = Client()
        cls.follower_user.force_login(cls.follower)
        cls.unfollower = User.objects.create_user(username='User')
        cls.unfollower_user = Client()

        cls.post = Post.objects.create(
            author=cls.author,
            text='текст'
        )

    def test_user_can_following(self):
        """Авторизованный пользователь может
        подписаться на другого пользователя"""
        count = Follow.objects.count()
        self.follower_user.get(reverse(
            'posts:profile_follow',
            kwargs={'username': self.author.username}))
        self.assertEqual(Follow.objects.count(), count + 1)

    def test_user_can_unfollowing(self):
        """Пользователь может отменить подписку на автора"""
        self.follower_user.get(reverse(
            'posts:profile_follow',
            kwargs={'username': self.author.username}))
        count = Follow.objects.count()
        self.follower_user.get(reverse(
            'posts:profile_unfollow',
            kwargs={'username': self.author.username}))
        self.assertEqual(Follow.objects.count(), count - 1)


# -*- coding: utf-8 -*-
#!/usr/bin/python

# TODO: DynamoDBに現在雨が降っているかの状態を登録し、メッセージの出し分けをする

import requests
from datetime import datetime, timedelta
import json
from pygooglechart import Chart
from pygooglechart import SimpleLineChart
from pygooglechart import Axis
import math
from pyshorteners import Shortener
import ConfigParser
from slackclient import SlackClient
import time
import boto3

inifile = ConfigParser.SafeConfigParser()
inifile.read("./config.ini")
GOOGLE_API_KEY = inifile.get('rainfall', 'google_api_key')
YAHOO_APP_ID = inifile.get('rainfall', 'yahoo_app_id')
LON = inifile.get('rainfall', 'lon')
LAT = inifile.get('rainfall', 'lat')
MAP_IMAGE_X = inifile.get('rainfall', 'map_image_x')
MAP_IMAGE_Y = inifile.get('rainfall', 'map_image_y')
ALERT_THRESH = float(inifile.get('rainfall', 'alert_thresh'))
ZOOM = inifile.get('rainfall', 'zoom')

def getTimeString(offset_minutes):
    d = datetime.now() + timedelta(hours=9, minutes=offset_minutes)
    return d.strftime("%H:%M")

def getRainfallRadarUrl(lat, lon, zoom, width, height):
    # http://developer.yahoo.co.jp/webapi/map/openlocalplatform/v1/static.html#exp_weather
    url = "http://map.olp.yahooapis.jp/OpenLocalPlatform/V1/static?appid=" + YAHOO_APP_ID + "&lat=" + str(lat) + "&lon=" + str(lon) + "&z=" + str(zoom) + "&width=" + str(width) + "&height=" + str(height) + "&overlay=type:rainfall"
    return url

def shorten_url(url):
    shortener = Shortener('Google', api_key=GOOGLE_API_KEY)
    return shortener.short(url)

def lambda_handler(event, context):
    # http://developer.yahoo.co.jp/webapi/map/openlocalplatform/v1/weather.html

    url = "http://weather.olp.yahooapis.jp/v1/place"

    payload = {'appid': YAHOO_APP_ID, 'coordinates': LON + "," + LAT, 'output': 'json'}
    result =  requests.get(url, params=payload)

    timeString = getTimeString(30)
    send_message = False

    json_data = json.loads(result.content)

    rainfall = json_data['Feature'][0]['Property']['WeatherList']['Weather']#[3]['Rainfall']

    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table(inifile.get('dynamodb', 'table_id'))
    key_id = int(inifile.get('dynamodb', 'id_value'))
    result = table.get_item(
        Key={
            'Id': key_id
        }
    )

    if 'raining' not in result:
        table.put_item(
            Item={
                "Id":key_id,
                "raining": False,
                "source": inifile.get('dynamodb', 'source')}
        )
    raining = result['Item']['raining']

    if rainfall[3]['Rainfall'] >= ALERT_THRESH and raining == False:
        send_message = True
        message = timeString + "で" + str(rainfall[3]['Rainfall']) + "mm/hの雨が予想されます。"
    elif rainfall[3]['Rainfall'] < ALERT_THRESH and raining == True:
        send_message = True
        message = timeString + "で" + "雨が止む予報です。"
    elif rainfall[3]['Rainfall'] >= ALERT_THRESH and raining == True:
        send_message = True
        message = timeString + "で" + str(rainfall[3]['Rainfall']) + "mm/hの雨が引き続き降る予報です。"
    elif rainfall[3]['Rainfall'] < ALERT_THRESH and raining == False:
        send_message = False

    if rainfall[3]['Rainfall'] >= ALERT_THRESH:
        table.put_item(
            Item={
                "Id":key_id,
                "raining": True,
                "source": inifile.get('dynamodb', 'source')}
        )
    else:
        table.put_item(
            Item={
                "Id":key_id,
                "raining": False,
                "source": inifile.get('dynamodb', 'source')}
        )

    if send_message == True:

        width = 500
        height = 150
        chart = SimpleLineChart(width, height)
        chart.set_colours(['719CFF'])

        rainfall_array = [rainfall[0]['Rainfall'], rainfall[1]['Rainfall'], rainfall[2]['Rainfall'], rainfall[3]['Rainfall'], rainfall[4]['Rainfall'], rainfall[5]['Rainfall'], rainfall[6]['Rainfall']]
        chart.add_data(rainfall_array)

        chart.set_axis_labels(Axis.BOTTOM, [getTimeString(0), getTimeString(10), getTimeString(20), getTimeString(30), getTimeString(40), getTimeString(50), getTimeString(60)])
        chart.set_legend(['Rainfall(mm/h)'])
        chart.set_legend_position('bv')

        if math.ceil(max(rainfall_array)) < 1.0:
            chart.y_range=(0, 1)
            chart.set_axis_range(Axis.LEFT, 0, 1)
        else:
            chart.y_range=(0, math.ceil(max(rainfall_array)))
            chart.set_axis_range(Axis.LEFT, 0, math.ceil(max(rainfall_array)))

        d = datetime.now() + timedelta(hours=9, minutes=30)
        date_str = d.strftime("|date:%Y%m%d%H%M|datelabel:on")

        radar_url = "http://weather.yahoo.co.jp/weather/zoomradar/?lat=" + LAT + "&lon=" + LON + "&z=12"
        rainfall_image_url = getRainfallRadarUrl(LAT, LON, ZOOM, MAP_IMAGE_X, MAP_IMAGE_Y) + date_str

        chart_url = shorten_url(chart.get_url())
        rainfall_image_url = shorten_url(rainfall_image_url)
        radar_url = shorten_url(radar_url)

        message = unicode(message, 'utf-8') + "\n" + chart_url + "\n" + timeString + u"の雨雲予想: " + rainfall_image_url + "\n" + u"詳細: " + radar_url + "\n"

        sc = SlackClient(inifile.get('slack', 'token'))
        res = sc.api_call("chat.postMessage", channel=inifile.get('slack', 'channel'), text=message, username=inifile.get('slack', 'username'), icon_emoji=inifile.get('slack', 'icon_emoji'))
        print(res)


from matplotlib.pyplot import annotate
import pandas as pd
from collections import defaultdict
from my_utils.common_utils import load_pkl_data, save_pkl_data
from transformers import AutoTokenizer
from transformers import AutoTokenizer, AutoModel, TapasTokenizer
import argparse
from tqdm import tqdm
import os
import re
import torch
from utils.annotation_processor import AnnotationProcessor, EvidenceType
from utils.wiki_page import WikiPage, get_wikipage_by_id,WikiTable
from database.feverous_db import FeverousDB
from fusion_col_parser import get_parser
import os
import dgl
import stanza

def connect_pooling_matrix(pooling_matrix):
    #cellids * 512
    if len(pooling_matrix) == 1:
        return pooling_matrix[0]
    left = pooling_matrix.pop(0)
    right = pooling_matrix[0]
    left_temp = torch.cat((left, torch.zeros(left.size()[0], right.size()[1])), dim = 1)
    right_temp = torch.cat((torch.zeros(right.size()[0], left.size()[1]), right), dim = 1)
    pooling_matrix[0] = torch.cat((left_temp, right_temp))
    return connect_pooling_matrix(pooling_matrix)

def collate_fn(batch):
    sent_input, sent_mask\
        , table_input_ids, table_attention_masks, table_token_type_ids\
        , fusion_graph, cell_graph\
        , col_pooling_matrix, pooling_matrix\
        , sent_labels, col_labels, cell_labels = map(list, zip(*batch))

    batch_sent_input = torch.cat(sent_input)
    batch_sent_mask = torch.cat(sent_mask)

    table_input_ids = [torch.reshape(item, (-1, 512)) for item in table_input_ids if item != []]
    table_attention_masks = [torch.reshape(item, (-1, 512)) for item in table_attention_masks if item != []]
    table_token_type_ids = [torch.reshape(item, (-1, 512, 7)) for item in table_token_type_ids if item != []]
    if table_input_ids != []:
        batch_table_ids = torch.cat(table_input_ids)
        batch_table_attention_masks = torch.cat(table_attention_masks)
        batch_table_token_type_ids = torch.cat(table_token_type_ids)

        pooling_matrix = [item for item in pooling_matrix if item != []]
        batch_pooling_matrix = connect_pooling_matrix(pooling_matrix)
        col_pooling_matrix = [item for item in col_pooling_matrix if item != []]
        batch_col_pooling_matrix = connect_pooling_matrix(col_pooling_matrix)
    else:
        batch_table_ids = torch.tensor([])
        batch_table_attention_masks = torch.tensor([])
        batch_table_token_type_ids = torch.tensor([])
        batch_pooling_matrix = torch.tensor([])
        batch_col_pooling_matrix = torch.tensor([])

    if len(fusion_graph) == 1:
        batch_graph = fusion_graph[0]
        batch_cell_graph = cell_graph[0]
    else:
        batch_graph = dgl.batch(fusion_graph)
        batch_cell_graph = dgl.batch(cell_graph)

    batch_sent_labels = torch.tensor([i for item in sent_labels for i in item])
    batch_cell_labels = torch.tensor([i for item in cell_labels for i in item])
    batch_col_labels = torch.tensor([i for item in col_labels for i in item])

    return batch_sent_input, batch_sent_mask\
        , batch_table_ids, batch_table_attention_masks, batch_table_token_type_ids\
        , batch_graph, batch_cell_graph\
        , batch_col_pooling_matrix, batch_pooling_matrix\
        , batch_sent_labels, batch_col_labels, batch_cell_labels

class FusionRetrievalDataset(torch.utils.data.Dataset):
    def __init__(self, data, args):
        self.data = data
        self.roberta_tokenizer = AutoTokenizer.from_pretrained(args.roberta_path)
        self.tapas_tokenizer = AutoTokenizer.from_pretrained(args.tapas_path)
        self.use_entity = args.use_entity_edges

    def __getitem__(self, idx):
        ### claim for roberta and tapas
    ### sentence for roberta
    ### table for tapas
    ### table_ids_2d for pooling
    ### sentence_id
    ### cell_id
    ### sentence labels
    ### cell labels
        claim = self.data['claim'][idx]
        sentences = self.data['sentences'][idx]
        tables = self.data['tables'][idx]
        table_ids_2D = self.data['table_ids_2D'][idx]
        cell_ids = self.data['cell_ids'][idx]
        sent_ids = self.data['sent_ids'][idx]
        sent_labels = self.data['sent_labels'][idx]
        col_labels = self.data['col_labels'][idx]
        cell_labels = self.data['cell_labels'][idx]

        claims = [claim]* len(sentences)

        sentence_input = self.roberta_tokenizer(claims, sentences, padding = 'max_length', max_length = 512, truncation = True, return_tensors = 'pt')

        table_input_ids = []
        table_attention_masks = []
        table_token_type_ids = []
        pooling_matrix = []
        col_pooling_matrix = []
        if len(cell_ids) > 0:
            for i in range(len(cell_ids)):
                temp_input, temp_mask, temp_token_type, temp_pooling_matrix, temp_col_pooling_matrix = self.tokenize_claim_and_inputs(claim, tables[i], cell_ids[i], table_ids_2D[i], self.tapas_tokenizer)
                table_input_ids.append(temp_input)
                table_attention_masks.append(temp_mask)
                table_token_type_ids.append(temp_token_type)
                pooling_matrix.append(temp_pooling_matrix)
                col_pooling_matrix.append(temp_col_pooling_matrix)
            table_input_ids = torch.stack(table_input_ids)
            table_attention_masks = torch.stack(table_attention_masks)
            table_token_type_ids = torch.stack(table_token_type_ids)
            col_pooling_matrix = connect_pooling_matrix(col_pooling_matrix)
            pooling_matrix = connect_pooling_matrix(pooling_matrix)

        # let's try fully connected graphs first
        if self.use_entity:
            fusion_graph_edges = list(self.data['sent_col_edges'][idx])
        else:
            fusion_graph_edges = connect_evi_in_two_sets(sent_ids + col_labels, sent_ids + col_labels)
            fusion_graph_edges = list(fusion_graph_edges)
        fusion_graph = dgl.graph(fusion_graph_edges)

        if self.use_entity:
            cell_graph_edges = list(self.data['cell_edges'][idx])
        else:
            cell_graph_edges = connect_evi_in_two_sets(cell_labels, cell_labels)
            cell_graph_edges = list(cell_graph_edges)
        cell_graph = dgl.graph(cell_graph_edges)

        fusion_graph = dgl.add_self_loop(fusion_graph)
        graph = dgl.add_self_loop(graph)
        if col_pooling_matrix != []:
            graph.ndata['t'] = torch.tensor([0] * len(sentences) + [1] * col_pooling_matrix.size()[0])
        else:
            graph.ndata['t'] = torch.tensor([0] * len(sentences))
        cell_graph = dgl.add_self_loop(cell_graph)
        return sentence_input['input_ids'], sentence_input['attention_mask']\
            , table_input_ids, table_attention_masks, table_token_type_ids\
                , fusion_graph, cell_graph\
                , col_pooling_matrix, pooling_matrix\
                , sent_labels, col_labels, cell_labels

    def __len__(self):
        return len(self.data['claim'])

    def tokenize_claim_and_inputs(self, claim, table, id_sequence, table_ids_2D, tokenizer):
        tokenized_inputs = tokenizer(table = table, queries = claim, padding = 'max_length', truncation = True, return_tensors = 'pt')
        pooling_matrix = torch.zeros([len(id_sequence), 512])
        col_pooling_matrix = torch.zeros([len(table_ids_2D[0]), 512])

        for i in range(512):
            if tokenized_inputs['token_type_ids'][0, i, 0] == 0:
                continue
            elif tokenized_inputs['token_type_ids'][0, i, 0] == 1 and  tokenized_inputs['token_type_ids'][0, i, 1] *  tokenized_inputs['token_type_ids'][0, i, 2] > 0:
                col_num = tokenized_inputs['token_type_ids'][0, i , 1] - 1
                col_pooling_matrix[col_num, i] = 1
        col_pooling_matrix = avearge_of_matrix(col_pooling_matrix)
        ##col_pooling_matrix = col_pooling_matrix

        for i in range(512):
            if tokenized_inputs['token_type_ids'][0, i, 0] == 0:
                continue
            elif  tokenized_inputs['token_type_ids'][0, i, 0] == 1 and  tokenized_inputs['token_type_ids'][0, i, 1] *  tokenized_inputs['token_type_ids'][0, i, 2] > 0:
                cell_id = table_ids_2D[tokenized_inputs['token_type_ids'][0, i, 2]-1][tokenized_inputs['token_type_ids'][0, i, 1]-1]
                if cell_id in id_sequence:
                    pooling_matrix[id_sequence.index(cell_id),i] = 1
        pooling_matrix = avearge_of_matrix(pooling_matrix)
        #average_num = torch.sum(pooling_matrix, dim = 1)
        #average_num = torch.reshape(average_num, (-1, 1))
        #average_num = torch.where(average_num == 0, torch.ones_like(average_num) , average_num)
        #pooling_matrix = torch.div(pooling_matrix, average_num)
        return tokenized_inputs['input_ids'], tokenized_inputs['attention_mask'], tokenized_inputs['token_type_ids'], pooling_matrix, col_pooling_matrix

    def collate_fn(cls):
        return collate_fn

def avearge_of_matrix(pooling_matrix):
    average_num = torch.sum(pooling_matrix, dim = 1)
    average_num = torch.reshape(average_num, (-1, 1))
    average_num = torch.where(average_num == 0, torch.ones_like(average_num), average_num)
    pooling_matrix = torch.div(pooling_matrix, average_num)
    return pooling_matrix

def connect_evi_in_two_sets(id_sequence1, id_sequence2):
    output_edge_set = set()
    for i in range(len(id_sequence1)):
        for j in range(len(id_sequence2)):
            output_edge_set.add((i, j))

    return output_edge_set

def build_edge_dict(sent_ids, cell_ids_2d):
    cell_ids = []
    for item in cell_ids_2d:
        cell_ids += item
    edge_dict = {}
    edge_dict[('sentence', 'sent2sent', 'sentence')] = connect_evi_in_two_sets(sent_ids, sent_ids)
    edge_dict[('sentence', 'sent2cell', 'cell')] = connect_evi_in_two_sets(sent_ids, cell_ids)
    edge_dict[('cell', 'sent2sent', 'sentence')] = connect_evi_in_two_sets(cell_ids, sent_ids)
    edge_dict[('cell', 'sent2sent', 'cell')] = connect_evi_in_two_sets(cell_ids, cell_ids)
    return edge_dict

def prepare_table(curr_tab, gold_evidence):
    page = curr_tab.page
    table_content_2D = []
    table_ids_2D = []
    col_labels = set()
    id_sequence = []
    for i, row in enumerate(curr_tab.rows):
        if i == 256:
            break
        row_id = []
        row_flat = []
        for j, cell in enumerate(row.row):
            if j == 128:
                break
            curr_id =page + '_' + cell.get_id()# get current cell id
            if curr_id in gold_evidence:
                col_labels.add(j)
            row_id.append(curr_id)# row id list
            row_flat.append(str(cell))# row content list

        table_ids_2D.append(row_id)
        table_content_2D.append(row_flat)

    col_labels = [1 if i in col_labels else 0 for i in range(len(table_ids_2D[0]))]
    table = pd.DataFrame(table_content_2D, columns=table_content_2D[0], dtype = str).fillna('')
    return table_ids_2D, table, col_labels

def clean_hyperlink_brakets(sentence):
    hyperlink = re.findall('\[\[.*?\|', sentence)
    hyperlink = [item[2:-1] for item in hyperlink]
    sentence = re.sub('\[\[.*?\|', '', sentence)
    sentence = re.sub('\]\]', '', sentence)
    return hyperlink, sentence

def build_dataset_batch(anno, args):
    ### This function is already checked
    ### This is the old function for fully connected graph
    ### return:
    ### id
    ### claim for roberta and tapas
    ### sentence for roberta
    ### table for tapas
    ### table_ids_2d for pooling
    ### edge_dict
    ### sentence_id
    ### cell_id
    ### sentence labels
    ### col_labes
    ### cell labels
    id = anno.id
    claim = anno.claim
    sentence_ids = []
    table_ids = set()
    cell_ids = []
    table_content = []
    table_ids_2D = []
    cell_id_by_table = defaultdict(list)
    gold_evidence = anno.get_evidence(flat = True)
    for evi in anno.predicted_evidence:
        if '_sentence_' in evi:
            sentence_ids.append(evi)
        elif '_table_' in evi:
            table_ids.add(evi)
        elif '_cell_' in evi:
            table_id = evi.split('_')[0] + '_table_' + evi.split('_')[-3]
            if table_id in table_ids:
                cell_id_by_table[table_id].append(evi)
    sentences = []
    for evi in sentence_ids:
        page_id = evi.split('_')[0]
        sent_id = '_'.join(evi.split('_')[1:])
        page_json = args.db.get_doc_json(page_id)
        curr_page = WikiPage(page_id, page_json)
        if curr_page is None:
            continue
        sentences.append(curr_page.get_element_by_id(sent_id).content)

    col_labels = []
    for evi in table_ids:
        cell_ids.append(cell_id_by_table[evi])
        page_id = evi.split('_')[0]
        table_id = '_'.join(evi.split('_')[1:])
        page_json = args.db.get_doc_json(page_id)
        curr_page = WikiPage(page_id, page_json)
        if curr_page is None:
            continue
        curr_table = curr_page.get_element_by_id(table_id)
        curr_id_2D, curr_content, curr_col_labels = prepare_table(curr_table, gold_evidence)
        table_content.append(curr_content)
        table_ids_2D.append(curr_id_2D)
        col_labels += curr_col_labels

    sentence_labels = [1 if evi in gold_evidence else 0 for evi in sentence_ids]
    cell_labels = [1 if evi in gold_evidence else 0  for item in cell_ids for evi in item]

    #edge_dict = build_edge_dict(sentence_ids, cell_ids)

    return id, claim, sentences, table_content, table_ids_2D, sentence_ids, cell_ids, sentence_labels, col_labels, cell_labels

def prepare_table_with_entity(curr_tab, gold_evidence, candidate_ids):
    page = curr_tab.page
    table_content_2D = []
    table_ids_2D = []
    col_labels = set()
    output = set()
    output_candidates = []
    candidates_content = []
    for i, row in enumerate(curr_tab.rows):
        if i == 256:
            break
        row_id = []
        row_flat = []
        for j, cell in enumerate(row.row):
            if j == 128:
                break
            curr_id =page + '_' + cell.get_id()# get current cell id
            if curr_id in gold_evidence:
                col_labels.add(j)
            row_id.append(curr_id)# row id list
            row_flat.append(str(cell))# row content list
            if curr_id in candidate_ids and curr_id not in output_candidates:
                output_candidates.append(curr_id)
                candidates_content.append(str(cell))

        table_ids_2D.append(row_id)
        table_content_2D.append(row_flat)

    col_labels = [1 if i in col_labels else 0 for i in range(len(table_ids_2D[0]))]
    table = pd.DataFrame(table_content_2D, columns=table_content_2D[0], dtype = str).fillna('')
    return table_ids_2D, table, col_labels, output_candidates, list(candidates_content)

def connect_in_one_set(evi_set, evi_ids):
    edge_set = set()
    for i in evi_set:
        for j in evi_set:
            edge_set.add((evi_ids.index(i),evi_ids.index(j)))
    return edge_set

def connect_evidence_in_pool(evidence_pool, sent_col_ids, cell_ids):
    sent_col_edges = set()
    cell_edges = set()
    sent_pool = set()
    cell_pool = set()
    for evi in evidence_pool:
        if '_sentence_' in evi:
            sent_pool.add(evi)
        if '_cell_' in evi:
            col_id_temp = evi.split('_')
            col_id = col_id_temp[0] + '_table_' + col_id_temp[-3] + '_col_' + col_id_temp[-1]
            sent_pool.add(col_id)
            cell_pool.add(evi)
    sent_col_edges = sent_col_edges.union(connect_in_one_set(sent_pool, sent_col_ids))
    cell_edges = cell_edges.union(connect_in_one_set(cell_pool, cell_ids))

    return sent_col_edges, cell_edges

def generate_col_id_sequence(table_ids, table_content_ids_2D):
    col_id_sequence = []
    for i, curr_tab_id in enumerate(table_ids):
        for j in range(len(table_content_ids_2D[i][0])):
            col_id_sequence.append(curr_tab_id +'_col_' + str(j))

    return col_id_sequence

def build_edges_based_on_entity(sentence_ids, cell_ids, entity_pool, hyperlink_pool, table_ids, table_content_ids_2D, cell2cell_page = True):
    ## entity pool: {entity: sentence or cell}
    ## hyperlink: {hyperlink: sentence or cell}
    ## evi_by_pages
    cell_ids = [id for item in cell_ids for id in item]
    sent_col_edges = set()
    cell_edges = set()
    sent_col_ids = sentence_ids + generate_col_id_sequence(table_ids, table_content_ids_2D)
    ### connect cell2cell col/sent2col/sent if in same pool
    ### connect sent with col if sent and its daughter cell share entity/hyperlink

    for key in entity_pool.keys():
        curr_sent_col_edges, curr_cell_edges = connect_evidence_in_pool(entity_pool[key], sent_col_ids, cell_ids)
        sent_col_edges = sent_col_edges.union(curr_sent_col_edges)
        cell_edges = cell_edges.union(curr_cell_edges)

    for key in hyperlink_pool.keys():
        curr_sent_col_edges, curr_cell_edges = connect_evidence_in_pool(hyperlink_pool[key], sent_col_ids, cell_ids)
        sent_col_edges = sent_col_edges.union(curr_sent_col_edges)
        cell_edges = cell_edges.union(curr_cell_edges)

    ### connect cell2cell if in same page
    ### DO NOT use this connection if you use all the cells as candiates!
    if cell2cell_page:
        evi_by_page = defaultdict(set)
        for evi_id in cell_ids:
            page = evi_id.split('_')[0]
            evi_by_page[page].add(evi_id)
        for page in evi_by_page.keys():
            curr_cell_edges = connect_in_one_set(evi_by_page[page], cell_ids)
            cell_edges = cell_edges.union(curr_cell_edges)
    ### connect sent2sent col2col sent2col if in same page
    evi_by_page = defaultdict(set)
    for evi_id in sent_col_ids:
        page = evi_id.split('_')[0]
        evi_by_page[page].add(evi_id)
    for page in evi_by_page.keys():
        curr_sent_col_edges = connect_in_one_set(evi_by_page[page], sent_col_ids)
        sent_col_edges = sent_col_edges.union(curr_sent_col_edges)
    return sent_col_edges, cell_edges

def build_dataset_with_entity_edges(anno, nlp, args):
     ### This function is already checked
    ### return:
    ### id
    ### claim for roberta and tapas
    ### sentence for roberta
    ### table for tapas
    ### table_ids_2d for pooling
    ### edge_dict
    ### sentence_id
    ### cell_id
    ### sentence labels
    ### col_labes
    ### cell labels
    id = anno.id
    claim = anno.claim
    sentence_ids = []
    table_ids = set()
    cell_ids = []
    table_content = []
    table_ids_2D = []
    cell_id_by_table = defaultdict(list)
    gold_evidence = anno.get_evidence(flat = True)
    for evi in anno.predicted_evidence:
        if '_sentence_' in evi:
            sentence_ids.append(evi)
        elif '_table_' in evi:
            table_ids.add(evi)

    sentences = []
    entity_pool = defaultdict(set)
    hyperlink_pool = defaultdict(set)
    sentence_ids = list(set(sentence_ids))
    for evi in sentence_ids:
        page_id = evi.split('_')[0]
        sent_id = '_'.join(evi.split('_')[1:])
        page_json = args.db.get_doc_json(page_id)
        curr_page = WikiPage(page_id, page_json)
        if curr_page is None:
            continue
        hyperlink, sentence = clean_hyperlink_brakets(curr_page.get_element_by_id(sent_id).content)
        for item in hyperlink:
            hyperlink_pool[item].add(evi)
        sentences.append(sentence)
    if sentences:
        in_docs = [stanza.Document([], text=d) for d in sentences]
        for i, sent in enumerate(nlp(in_docs)):
            if sent.sentences:
                for token in sent.sentences[0].tokens:
                    if token.ner != 'O':
                        entity_pool[token.text].add(sentence_ids[i])

    col_labels = []
    for evi in table_ids:
        curr_cell_ids = cell_id_by_table[evi]
        page_id = evi.split('_')[0]
        table_id = '_'.join(evi.split('_')[1:])
        page_json = args.db.get_doc_json(page_id)
        curr_page = WikiPage(page_id, page_json)
        if curr_page is None:
            continue
        curr_table = curr_page.get_element_by_id(table_id)
        curr_id_2D, curr_content, curr_col_labels, curr_cell_ids, curr_cell_content = prepare_table_with_entity(curr_table, gold_evidence, curr_cell_ids)
        for i, content in enumerate(curr_cell_content):
            hyperlink, content = clean_hyperlink_brakets(content)
            for item in hyperlink:
                hyperlink_pool[item].add(curr_cell_ids[i])
            curr_cell_content[i] = content
        if curr_cell_content:
            in_docs = [stanza.Document([], text=d) for d in curr_cell_content]
            for i, sent in enumerate(nlp(in_docs)):
                if sent.sentences:
                    for token in sent.sentences[0].tokens:
                        if token.ner != 'O':
                            entity_pool[token.text].add(curr_cell_ids[i])
        table_content.append(curr_content)
        table_ids_2D.append(curr_id_2D)
        col_labels.append(curr_col_labels)
        cell_ids.append(curr_cell_ids)

    sent_col_edges, cell_edges = build_edges_based_on_entity(sentence_ids, cell_ids, entity_pool, hyperlink_pool, table_ids, table_ids_2D)

    sentence_labels = [1 if evi in gold_evidence else 0 for evi in sentence_ids]
    cell_labels = [1 if evi in gold_evidence else 0  for item in cell_ids for evi in item]
    col_labels = [evi_label for item in col_labels for evi_label in item]

    return id, claim, sentences, table_content, table_ids_2D, sentence_ids, cell_ids, sentence_labels, col_labels, cell_labels, sent_col_edges, cell_edges

def build_dataset(split, args, cache_file = None, train_mode = True, use_entity = True):

    if os.path.exists(cache_file):
        print('Data Loading From ' + cache_file +' ...')
        output = load_pkl_data(cache_file)
        return output

    anno_file = '{0}/{1}.roberta_sent.table.not_precomputed.p{2}.s{3}.t{4}.jsonl'.format(args.data_path, split, args.max_page, args.max_sent, args.max_tabs)

    output = defaultdict(list)
    anno_processor = AnnotationProcessor(anno_file)
    annotations = [anno for anno in anno_processor]
    nlp = stanza.Pipeline('en', processors="tokenize,ner", verbose=False, tokenize_pretokenized=True, download_method= None)
    for i, anno in tqdm(enumerate(annotations), desc = 'build dataset at ' + cache_file):
    ### claim for roberta and tapas
    ### sentence for roberta
    ### table for tapas
    ### table_ids_2d for pooling
    ### edge_dict
    ### sentence_id
    ### cell_id
    ### sentence labels
    ### col labels
    ### cell labels

    ### filter
    ### filter out data only with retrieved sentence or tables
    ### Use data containing both format of data in training.
    ### the ids are collected. These can be used to construct graphs afterwards

        id, claim, sentences, table_content, table_ids_2D, sentence_ids, cell_ids, sentence_labels, col_labels, cell_labels, sent_col_edges, cell_edges = build_dataset_with_entity_edges(anno, nlp, args)
        if not sentences or not table_content or not cell_labels:
            continue
        if 1 not in sentence_labels and 1 not in cell_labels:
            if train_mode:
                continue
        output['id'].append(id)
        output['claim'].append(claim)
        output['sentences'].append(sentences)
        output['tables'].append(table_content)
        output['table_ids_2D'].append(table_ids_2D)
        output['sent_ids'].append(sentence_ids)
        output['cell_ids'].append(cell_ids)
        output['sent_labels'].append(sentence_labels)
        output['col_labels'].append(col_labels)
        output['cell_labels'].append(cell_labels)
        output['sent_col_edges'].append(sent_col_edges)
        output['cell_edges'].append(cell_edges)

    if cache_file:
        save_pkl_data(output, cache_file)

    return output

def check_cell_graph(data):
    ### This function is for debugging
    ### You may encounter this bug during training
    ### dgl._ffi.base.DGLError: There are 0-in-degree nodes in the graph, output for those nodes will be invalid.
    ### Reason could be:
    ##### some ids are repeated in the id_sequence, causing no in-degree and redutant nodes
    ### This function is to localate the bug
    for i in tqdm(range(len(data['id']))):
        if i < 45000:
            continue
        cell_graph_edges = data['cell_edges'][i]
        cell_graph = dgl.graph(list(cell_graph_edges))
        cell_ids = data['cell_ids'][i]
        cell_ids = [item for evi in cell_ids for item in evi]

        if 0 in cell_graph.in_degrees():
            print('Found 0 in degrees at cell graph'+ str(data['id'][i]))
            print('Index ' + str(i))
            show_data_for_debugging(data, i)
        if cell_graph.num_nodes() != len(cell_ids):
            print('num nodes not match')
            print(len(cell_ids))
            print(cell_graph.num_nodes())
            print(data['id'][i])
            print(i)

def show_data_for_debugging(data, idx):
    print(data['id'][idx])
    print(data['table_ids_2D'][idx])
    print(data['tables'][idx])
    print(data['cell_ids'][idx])
    print(len(data['cell_labels'][idx]))
    print(data['cell_labels'][idx])
    cell_graph = dgl.graph(list(data['cell_edges'][idx]))
    print(cell_graph.num_nodes())
    print(cell_graph.in_degrees())

if __name__ == '__main__':
    args = get_parser().parse_args()
    args.db = FeverousDB(args.wiki_path)

    output = build_dataset(args.split , args, cache_file='{0}/{1}_entity_graph.jsonl', train_mode= (args.split == 'train'))