#Make sure to install the following packages: dlt, langchain, duckdb, python-dotenv, openai, weaviate-client

import dlt
from langchain import PromptTemplate, LLMChain
from langchain.chains.openai_functions import create_structured_output_chain
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import PyPDFLoader
import weaviate
import os
import json

import argparse

from langchain.embeddings import OpenAIEmbeddings
from langchain.prompts import HumanMessagePromptTemplate, ChatPromptTemplate
from langchain.retrievers import WeaviateHybridSearchRetriever
from langchain.schema import Document, SystemMessage, HumanMessage
from langchain.vectorstores import Weaviate
import uuid
from dotenv import load_dotenv
load_dotenv()
from pathlib import Path
from langchain import OpenAI, LLMMathChain

import os

embeddings = OpenAIEmbeddings()

from deep_translator import (GoogleTranslator)

def _convert_pdf_to_document(path: str = None):

    """Convert a PDF document to a Document object"""
    if path is None:
        raise ValueError("A valid path to the document must be provided.")

    loader = PyPDFLoader(path)
    pages = loader.load_and_split()

    print("PAGES", pages[0])

    # Parse metadata from the folder path
    path_parts = Path(path).parts
    personal_receipts_index = path_parts.index("personal_receipts")
    metadata_parts = path_parts[personal_receipts_index+1:]

    documents = []
    for page in pages:
        translation = GoogleTranslator(source='auto', target='en').translate(text=page.page_content)
        documents.append(
            Document(
                metadata={
                    "title": "Personal Receipt",
                    "country": metadata_parts[1],
                    "year": metadata_parts[0],
                    "author": str(uuid.uuid4()),
                    "source": "/".join(metadata_parts),
                },
                page_content=translation,
            )
        )
    print(documents)

    return documents



def _init_weaviate():
    """Initialize weaviate client and retriever"""
    auth_config = weaviate.auth.AuthApiKey(api_key=os.environ.get('WEAVIATE_API_KEY'))
    client = weaviate.Client(
        url='https://my-vev-index-o4qitptw.weaviate.network',
        auth_client_secret=auth_config,

        additional_headers={
            "X-OpenAI-Api-Key": os.environ.get('OPENAI_API_KEY')
        }
    )
    retriever = WeaviateHybridSearchRetriever(
        client=client,
        index_name="PDFloader",
        text_key="text",
        attributes=[],
        embedding=embeddings,
        create_schema_if_missing=True,
    )

    return retriever
def load_to_weaviate(document_path=None):
    """Load documents to weaviate"""
    retriever =_init_weaviate()

    docs = _convert_pdf_to_document(document_path)

    return retriever.add_documents(docs)


def get_from_weaviate(query=None, path=None, operator=None, valueText=None):
    """
    Get documents from weaviate.

    Args:
        query (str): The query string.
        path (list): The path for filtering, e.g., ['year'].
        operator (str): The operator for filtering, e.g., 'Equal'.
        valueText (str): The value for filtering, e.g., '2017*'.

    Example:
        get_from_weaviate(query="some query", path=['year'], operator='Equal', valueText='2017*')
    """
    retriever = _init_weaviate()

    # Initial retrieval without filters
    output = retriever.get_relevant_documents(
        query,
        score=True,
    )

    # Apply filters if provided
    if path or operator or valueText:
        # Create the where_filter based on provided parameters
        where_filter = {
            'path': path if path else [],
            'operator': operator if operator else '',
            'valueText': valueText if valueText else ''
        }

        # Retrieve documents with filters applied
        output = retriever.get_relevant_documents(
            query,
            score=True,
            where_filter=where_filter
        )

    return output


def delete_from_weaviate(query=None, filters=None):
    """Delete documents from weaviate, pass dict as filters"""
    """  {
        'path': ['year'],
        'operator': 'Equal',
        'valueText': '2017*'     }"""
    auth_config = weaviate.auth.AuthApiKey(api_key=os.environ.get('WEAVIATE_API_KEY'))
    client = weaviate.Client(
        url='https://my-vev-index-o4qitptw.weaviate.network',
        auth_client_secret=auth_config,

        additional_headers={
            "X-OpenAI-Api-Key": os.environ.get('OPENAI_API_KEY')
        }
    )
    client.batch.delete_objects(
        class_name='PDFloader',
        # Same `where` filter as in the GraphQL API
        where={
            'path': ['year'],
            'operator': 'Equal',
            'valueText': '2017*'
        },
    )

    return "Success"


llm = ChatOpenAI(
            temperature=0.0,
            max_tokens=1200,
            openai_api_key=os.environ.get('OPENAI_API_KEY'),
            model_name="gpt-4-0613",
        )



def infer_schema_from_text(text: str):
    """Infer schema from text"""

    prompt_ = """ You are a json schema master. Create a JSON schema based on the following data and don't write anything else: {prompt} """

    complete_query = PromptTemplate(
    input_variables=["prompt"],
    template=prompt_,
)

    chain = LLMChain(
        llm=llm, prompt=complete_query, verbose=True
    )
    chain_result = chain.run(prompt=text).strip()

    json_data = json.dumps(chain_result)
    return json_data


def set_data_contract(data, version, date, agreement_id=None, privacy_policy=None, terms_of_service=None, format=None, schema_version=None, checksum=None, owner=None, license=None, validity_start=None, validity_end=None):
    # Creating the generic data contract

    data_contract = {
        "version": version or "",
        "date": date or "",
        "agreement_id": agreement_id or "",
        "privacy_policy": privacy_policy or "",
        "terms_of_service": terms_of_service or "",
        "format": format or "",
        "schema_version": schema_version or "",
        "checksum": checksum or "",
        "owner": owner or "",
        "license": license or "",
        "validity_start": validity_start or "",
        "validity_end": validity_end or "",
        "properties": data  # Adding the given data under the "properties" field
    }



    return data_contract

def create_id_dict(memory_id=None, st_memory_id=None, buffer_id=None):
    """
    Create a dictionary containing IDs for memory, st_memory, and buffer.

    Args:
        memory_id (str): The Memory ID.
        st_memory_id (str): The St_memory ID.
        buffer_id (str): The Buffer ID.

    Returns:
        dict: A dictionary containing the IDs.
    """
    id_dict = {
        "memoryID": memory_id or "",
        "st_MemoryID": st_memory_id or "",
        "bufferID": buffer_id or ""
    }
    return id_dict



def init_buffer(data, version, date, memory_id=None, st_memory_id=None, buffer_id=None, agreement_id=None, privacy_policy=None, terms_of_service=None, format=None, schema_version=None, checksum=None, owner=None, license=None, validity_start=None, validity_end=None, text=None, process=None):
    # Create ID dictionary
    id_dict = create_id_dict(memory_id, st_memory_id, buffer_id)

    # Set data contract
    data_contract = set_data_contract(data, version, date, agreement_id, privacy_policy, terms_of_service, format, schema_version, checksum, owner, license, validity_start, validity_end)

    # Add ID dictionary to properties
    data_contract["properties"]["relations"] = id_dict

    # Infer schema from text and add to properties
    if text:
        schema = infer_schema_from_text(text)
        data_contract["properties"]["schema"] = schema

    if process:
        data_contract["properties"]["process"] = process


    return data_contract


def infer_properties_from_text(text: str):
    """Infer schema properties from text"""

    prompt_ = """ You are a json index master. Create a short JSON index containing the most important data and don't write anything else: {prompt} """

    complete_query = PromptTemplate(
    input_variables=["prompt"],
    template=prompt_,
)

    chain = LLMChain(
        llm=llm, prompt=complete_query, verbose=True
    )
    chain_result = chain.run(prompt=text).strip()
    # json_data = json.dumps(chain_result)
    return chain_result
#
#
# # print(infer_schema_from_text(output[0].page_content))


def load_json_or_infer_schema(file_path, document_path):
    """Load JSON schema from file or infer schema from text"""
    try:
        # Attempt to load the JSON file
        with open(file_path, 'r') as file:
            json_schema = json.load(file)
        return json_schema
    except FileNotFoundError:
        # If the file doesn't exist, run the specified function
        output = _convert_pdf_to_document(path=document_path)
        json_schema = infer_schema_from_text(output[0].page_content)
        return json_schema




def ai_function(prompt=None, json_schema=None):
    """AI function to convert unstructured data to structured data"""
    # Here we define the user prompt and the structure of the output we desire
    # prompt = output[0].page_content

    prompt_msgs = [
        SystemMessage(
            content="You are a world class algorithm converting unstructured data into structured data."
        ),
        HumanMessage(content="Convert unstructured data to structured data:"),
        HumanMessagePromptTemplate.from_template("{input}"),
        HumanMessage(content="Tips: Make sure to answer in the correct format"),
    ]
    prompt_ = ChatPromptTemplate(messages=prompt_msgs)
    chain = create_structured_output_chain(json_schema , prompt=prompt_, llm=llm, verbose=True)
    output = chain.run(input = prompt, llm=llm)
    yield output


# Define a base directory if you have one; this could be the directory where your script is located
BASE_DIR = os.path.dirname(os.path.abspath(__file__))



def higher_level_thinking():
    """Higher level thinking function to calculate the sum of the price of the tickets from these documents"""

    docs_data = get_from_weaviate(query="Train", path=['year'], operator='Equal', valueText='2017*')
    str_docs_data = str(docs_data)

    llm_math = LLMMathChain.from_llm(llm, verbose=True)
    output = llm_math.run(f"Calculate the sum of the price of the tickets from these documents: {str_docs_data}")

    # data_format = init_buffer(data=output, version="0.0.1", date="2021-09-01")
    yield output
result_higher_level_thinking = higher_level_thinking()
def process_higher_level_thinking(result=None):
    data_format = init_buffer(data=result, version="0.0.1", date="2021-09-01")
    import json
    data_format=json.dumps(data_format)
    yield data_format

document_paths = [
    os.path.join(BASE_DIR, "personal_receipts", "2017", "de", "public_transport", "3ZCCCW.pdf"),
    os.path.join(BASE_DIR, "personal_receipts", "2017", "de", "public_transport", "4GBEC9.pdf")
]



def main(raw_loading, processed_loading,document_paths):
    BASE_DIR = os.getcwd()  # Assuming the current working directory is where the data_processing_script.py is located

    def format_document_paths(base_dir, path):
        # Split the input path and extract the elements
        elements = path.strip("/").split("/")

        # Construct the document_paths list
        document_paths = [os.path.join(base_dir, *elements)]

        return document_paths

    document_paths_ =[format_document_paths(BASE_DIR, path) for path in document_paths][0]
    print(document_paths)

    if raw_loading:
        for document in document_paths_:
            file_path = os.path.join(BASE_DIR, "ticket_schema.json")
            json_schema = load_json_or_infer_schema(file_path, document)
            output = _convert_pdf_to_document(path=document)
            find_data_in_store = get_from_weaviate(query="Train", path=['year'], operator='Equal', valueText='2017*')

            if find_data_in_store:
                output = find_data_in_store
                print(output[1])
            else:
                load_to_weaviate(document)

            pipeline = dlt.pipeline(pipeline_name="train_ticket", destination='duckdb', dataset_name='train_ticket_data')
            info = pipeline.run(data=ai_function(output[0].page_content, json_schema))
            print(info)

    elif processed_loading:
        pipeline_processed = dlt.pipeline(pipeline_name="train_ticket_processed", destination='duckdb',
                                          dataset_name='train_ticket_processed_data')
        info = pipeline_processed.run(data=higher_level_thinking())
        print(info)

    else:
        print("Please specify either '--raw_loading' or '--processed_loading' option.")




if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Data Processing Script")
    parser.add_argument("--raw_loading", action="store_true", help="Load raw document data and perform AI tasks")
    parser.add_argument("--processed_loading", action="store_true",
                        help="Load processed data and run higher-level thinking AI function")
    parser.add_argument("document_paths", nargs="*", help="Paths to the documents to process")

    args = parser.parse_args()

    main(args.raw_loading, args.processed_loading, args.document_paths)


#to run:  python3 level_1_pdf_vectorstore_dlt_etl.py --raw_loading "/personal_receipts/2017/de/public_transport/3ZCCCW.pdf"


langchain.LLMChain langchain.chains.openai_functions.create_structured_output_chain langchain.prompts.HumanMessagePromptTemplate.from_template langchain.retrievers.WeaviateHybridSearchRetriever langchain.LLMMathChain.from_llm langchain.schema.HumanMessage langchain.schema.SystemMessage langchain.prompts.ChatPromptTemplate langchain.document_loaders.PyPDFLoader langchain.embeddings.OpenAIEmbeddings langchain.PromptTemplate
[((741, 754), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (752, 754), False, 'from dotenv import load_dotenv\n'), ((848, 866), 'langchain.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (864, 866), False, 'from langchain.embeddings import OpenAIEmbeddings\n'), ((1129, 1146), 'langchain.document_loaders.PyPDFLoader', 'PyPDFLoader', (['path'], {}), '(path)\n', (1140, 1146), False, 'from langchain.document_loaders import PyPDFLoader\n'), ((2438, 2599), 'langchain.retrievers.WeaviateHybridSearchRetriever', 'WeaviateHybridSearchRetriever', ([], {'client': 'client', 'index_name': '"""PDFloader"""', 'text_key': '"""text"""', 'attributes': '[]', 'embedding': 'embeddings', 'create_schema_if_missing': '(True)'}), "(client=client, index_name='PDFloader',\n    text_key='text', attributes=[], embedding=embeddings,\n    create_schema_if_missing=True)\n", (2467, 2599), False, 'from langchain.retrievers import WeaviateHybridSearchRetriever\n'), ((5296, 5356), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['prompt']", 'template': 'prompt_'}), "(input_variables=['prompt'], template=prompt_)\n", (5310, 5356), False, 'from langchain import PromptTemplate, LLMChain\n'), ((5381, 5435), 'langchain.LLMChain', 'LLMChain', ([], {'llm': 'llm', 'prompt': 'complete_query', 'verbose': '(True)'}), '(llm=llm, prompt=complete_query, verbose=True)\n', (5389, 5435), False, 'from langchain import PromptTemplate, LLMChain\n'), ((5517, 5541), 'json.dumps', 'json.dumps', (['chain_result'], {}), '(chain_result)\n', (5527, 5541), False, 'import json\n'), ((8185, 8245), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['prompt']", 'template': 'prompt_'}), "(input_variables=['prompt'], template=prompt_)\n", (8199, 8245), False, 'from langchain import PromptTemplate, LLMChain\n'), ((8270, 8324), 'langchain.LLMChain', 'LLMChain', ([], {'llm': 'llm', 'prompt': 'complete_query', 'verbose': '(True)'}), '(llm=llm, prompt=complete_query, verbose=True)\n', (8278, 8324), False, 'from langchain import PromptTemplate, LLMChain\n'), ((9693, 9733), 'langchain.prompts.ChatPromptTemplate', 'ChatPromptTemplate', ([], {'messages': 'prompt_msgs'}), '(messages=prompt_msgs)\n', (9711, 9733), False, 'from langchain.prompts import HumanMessagePromptTemplate, ChatPromptTemplate\n'), ((9746, 9832), 'langchain.chains.openai_functions.create_structured_output_chain', 'create_structured_output_chain', (['json_schema'], {'prompt': 'prompt_', 'llm': 'llm', 'verbose': '(True)'}), '(json_schema, prompt=prompt_, llm=llm,\n    verbose=True)\n', (9776, 9832), False, 'from langchain.chains.openai_functions import create_structured_output_chain\n'), ((10024, 10049), 'os.path.abspath', 'os.path.abspath', (['__file__'], {}), '(__file__)\n', (10039, 10049), False, 'import os\n'), ((10347, 10387), 'langchain.LLMMathChain.from_llm', 'LLMMathChain.from_llm', (['llm'], {'verbose': '(True)'}), '(llm, verbose=True)\n', (10368, 10387), False, 'from langchain import OpenAI, LLMMathChain\n'), ((10815, 10838), 'json.dumps', 'json.dumps', (['data_format'], {}), '(data_format)\n', (10825, 10838), False, 'import json\n'), ((10885, 10980), 'os.path.join', 'os.path.join', (['BASE_DIR', '"""personal_receipts"""', '"""2017"""', '"""de"""', '"""public_transport"""', '"""3ZCCCW.pdf"""'], {}), "(BASE_DIR, 'personal_receipts', '2017', 'de',\n    'public_transport', '3ZCCCW.pdf')\n", (10897, 10980), False, 'import os\n'), ((10982, 11077), 'os.path.join', 'os.path.join', (['BASE_DIR', '"""personal_receipts"""', '"""2017"""', '"""de"""', '"""public_transport"""', '"""4GBEC9.pdf"""'], {}), "(BASE_DIR, 'personal_receipts', '2017', 'de',\n    'public_transport', '4GBEC9.pdf')\n", (10994, 11077), False, 'import os\n'), ((11151, 11162), 'os.getcwd', 'os.getcwd', ([], {}), '()\n', (11160, 11162), False, 'import os\n'), ((12901, 12962), 'argparse.ArgumentParser', 'argparse.ArgumentParser', ([], {'description': '"""Data Processing Script"""'}), "(description='Data Processing Script')\n", (12924, 12962), False, 'import argparse\n'), ((1273, 1283), 'pathlib.Path', 'Path', (['path'], {}), '(path)\n', (1277, 1283), False, 'from pathlib import Path\n'), ((4976, 5008), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (4990, 5008), False, 'import os\n'), ((9321, 9438), 'langchain.schema.SystemMessage', 'SystemMessage', ([], {'content': '"""You are a world class algorithm converting unstructured data into structured data."""'}), "(content=\n    'You are a world class algorithm converting unstructured data into structured data.'\n    )\n", (9334, 9438), False, 'from langchain.schema import Document, SystemMessage, HumanMessage\n'), ((9460, 9529), 'langchain.schema.HumanMessage', 'HumanMessage', ([], {'content': '"""Convert unstructured data to structured data:"""'}), "(content='Convert unstructured data to structured data:')\n", (9472, 9529), False, 'from langchain.schema import Document, SystemMessage, HumanMessage\n'), ((9539, 9590), 'langchain.prompts.HumanMessagePromptTemplate.from_template', 'HumanMessagePromptTemplate.from_template', (['"""{input}"""'], {}), "('{input}')\n", (9579, 9590), False, 'from langchain.prompts import HumanMessagePromptTemplate, ChatPromptTemplate\n'), ((9600, 9671), 'langchain.schema.HumanMessage', 'HumanMessage', ([], {'content': '"""Tips: Make sure to answer in the correct format"""'}), "(content='Tips: Make sure to answer in the correct format')\n", (9612, 9671), False, 'from langchain.schema import Document, SystemMessage, HumanMessage\n'), ((2143, 2177), 'os.environ.get', 'os.environ.get', (['"""WEAVIATE_API_KEY"""'], {}), "('WEAVIATE_API_KEY')\n", (2157, 2177), False, 'import os\n'), ((4324, 4358), 'os.environ.get', 'os.environ.get', (['"""WEAVIATE_API_KEY"""'], {}), "('WEAVIATE_API_KEY')\n", (4338, 4358), False, 'import os\n'), ((8758, 8773), 'json.load', 'json.load', (['file'], {}), '(file)\n', (8767, 8773), False, 'import json\n'), ((11476, 11509), 'os.path.join', 'os.path.join', (['base_dir', '*elements'], {}), '(base_dir, *elements)\n', (11488, 11509), False, 'import os\n'), ((11746, 11790), 'os.path.join', 'os.path.join', (['BASE_DIR', '"""ticket_schema.json"""'], {}), "(BASE_DIR, 'ticket_schema.json')\n", (11758, 11790), False, 'import os\n'), ((12241, 12343), 'dlt.pipeline', 'dlt.pipeline', ([], {'pipeline_name': '"""train_ticket"""', 'destination': '"""duckdb"""', 'dataset_name': '"""train_ticket_data"""'}), "(pipeline_name='train_ticket', destination='duckdb',\n    dataset_name='train_ticket_data')\n", (12253, 12343), False, 'import dlt\n'), ((12509, 12631), 'dlt.pipeline', 'dlt.pipeline', ([], {'pipeline_name': '"""train_ticket_processed"""', 'destination': '"""duckdb"""', 'dataset_name': '"""train_ticket_processed_data"""'}), "(pipeline_name='train_ticket_processed', destination='duckdb',\n    dataset_name='train_ticket_processed_data')\n", (12521, 12631), False, 'import dlt\n'), ((1483, 1527), 'deep_translator.GoogleTranslator', 'GoogleTranslator', ([], {'source': '"""auto"""', 'target': '"""en"""'}), "(source='auto', target='en')\n", (1499, 1527), False, 'from deep_translator import GoogleTranslator\n'), ((2373, 2405), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (2387, 2405), False, 'import os\n'), ((4554, 4586), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (4568, 4586), False, 'import os\n'), ((1817, 1829), 'uuid.uuid4', 'uuid.uuid4', ([], {}), '()\n', (1827, 1829), False, 'import uuid\n')]