import tempfile
import time
import os
from utils import compute_sha1_from_file
from langchain.schema import Document
import streamlit as st
from langchain.text_splitter import RecursiveCharacterTextSplitter
from typing import List
from sqlite3 import Connection
from verse.sqlite_helper import *


def update_metadata(conn: Connection, docs_with_metadata: List[Document]):
    insert_tuple = list(
        set(
            map(
                lambda x: (
                    hash(x.metadata["file_sha1"]),
                    x.metadata["file_sha1"],
                    x.metadata["file_name"],
                ),
                docs_with_metadata,
            )
        )
    )
    insertmany(conn=conn, datalist=insert_tuple)


def process_file(
    conn: Connection, file, loader_class, file_suffix, stats_db=None
) -> List[Document]:
    documents = []
    file_name = file.name
    file_size = file.size
    if st.secrets.self_hosted == "false":
        if file_size > 1000000:
            st.error(
                "File size is too large. Please upload a file smaller than 1MB or self host."
            )
            return

    dateshort = time.strftime("%Y%m%d")
    with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
        tmp_file.write(file.getvalue())
        tmp_file.flush()

        loader = loader_class(tmp_file.name)
        documents = loader.load()
        file_sha1 = compute_sha1_from_file(tmp_file.name)

    os.remove(tmp_file.name)

    chunk_size = st.session_state["chunk_size"]
    chunk_overlap = st.session_state["chunk_overlap"]
    print(f"Chunk Size {chunk_size} Overlap {chunk_overlap}")

    text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
        chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n\n", ""]
    )
    documents = text_splitter.split_documents(documents)

    # Add the document sha1 as metadata to each document
    docs_with_metadata = [
        Document(
            page_content=doc.page_content,
            metadata={
                "file_sha1": file_sha1,
                "file_size": file_size,
                "file_name": file_name,
                "chunk_size": chunk_size,
                "chunk_overlap": chunk_overlap,
                "date": dateshort,
                "file_type": file_suffix,
                "page": doc.metadata["page"],
                "dbsource": doc.metadata["source"]
            },
        )
        for doc in documents
    ]

    return docs_with_metadata

langchain.text_splitter.RecursiveCharacterTextSplitter.from_tiktoken_encoder langchain.schema.Document
[((1152, 1175), 'time.strftime', 'time.strftime', (['"""%Y%m%d"""'], {}), "('%Y%m%d')\n", (1165, 1175), False, 'import time\n'), ((1468, 1492), 'os.remove', 'os.remove', (['tmp_file.name'], {}), '(tmp_file.name)\n', (1477, 1492), False, 'import os\n'), ((1679, 1812), 'langchain.text_splitter.RecursiveCharacterTextSplitter.from_tiktoken_encoder', 'RecursiveCharacterTextSplitter.from_tiktoken_encoder', ([], {'chunk_size': 'chunk_size', 'chunk_overlap': 'chunk_overlap', 'separators': "['\\n\\n', '']"}), "(chunk_size=chunk_size,\n    chunk_overlap=chunk_overlap, separators=['\\n\\n', ''])\n", (1731, 1812), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1185, 1246), 'tempfile.NamedTemporaryFile', 'tempfile.NamedTemporaryFile', ([], {'delete': '(False)', 'suffix': 'file_suffix'}), '(delete=False, suffix=file_suffix)\n', (1212, 1246), False, 'import tempfile\n'), ((1425, 1462), 'utils.compute_sha1_from_file', 'compute_sha1_from_file', (['tmp_file.name'], {}), '(tmp_file.name)\n', (1447, 1462), False, 'from utils import compute_sha1_from_file\n'), ((1973, 2281), 'langchain.schema.Document', 'Document', ([], {'page_content': 'doc.page_content', 'metadata': "{'file_sha1': file_sha1, 'file_size': file_size, 'file_name': file_name,\n    'chunk_size': chunk_size, 'chunk_overlap': chunk_overlap, 'date':\n    dateshort, 'file_type': file_suffix, 'page': doc.metadata['page'],\n    'dbsource': doc.metadata['source']}"}), "(page_content=doc.page_content, metadata={'file_sha1': file_sha1,\n    'file_size': file_size, 'file_name': file_name, 'chunk_size':\n    chunk_size, 'chunk_overlap': chunk_overlap, 'date': dateshort,\n    'file_type': file_suffix, 'page': doc.metadata['page'], 'dbsource': doc\n    .metadata['source']})\n", (1981, 2281), False, 'from langchain.schema import Document\n'), ((998, 1095), 'streamlit.error', 'st.error', (['"""File size is too large. Please upload a file smaller than 1MB or self host."""'], {}), "(\n    'File size is too large. Please upload a file smaller than 1MB or self host.'\n    )\n", (1006, 1095), True, 'import streamlit as st\n')]