import os
import openai
from dotenv import load_dotenv
import logging
import re
import hashlib

from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.llms import AzureOpenAI
from langchain.vectorstores.base import VectorStore
from langchain.chains import ChatVectorDBChain
from langchain.chains import ConversationalRetrievalChain
from langchain.chains.qa_with_sources import load_qa_with_sources_chain
from langchain.chains.llm import LLMChain
from langchain.chains.chat_vector_db.prompts import CONDENSE_QUESTION_PROMPT
from langchain.prompts import PromptTemplate
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders import WebBaseLoader
from langchain.text_splitter import TokenTextSplitter, TextSplitter
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders import TextLoader
from langchain.chat_models import ChatOpenAI
from langchain.schema import AIMessage, HumanMessage, SystemMessage
from langchain.agents import initialize_agent, Tool, AgentType

from utilities.formrecognizer import AzureFormRecognizerClient
from utilities.azureblobstorage import AzureBlobStorageClient
from utilities.translator import AzureTranslatorClient
from utilities.customprompt import PROMPT
from utilities.redis import RedisExtended
from utilities.azuresearch import AzureSearch
from utilities.NewAzureOpenAI import NewAzureOpenAI
import langchain

import pandas as pd
import urllib

from fake_useragent import UserAgent

from utilities.tools import LifeKnowledgeSearchTool, IotDeviceControlTool, MansionPriceTool, LifeKnowledgeSearchConfig

langchain.verbose = True

class LLMHelper:
    def __init__(self,
        document_loaders : BaseLoader = None, 
        text_splitter: TextSplitter = None,
        embeddings: OpenAIEmbeddings = None,
        llm: AzureOpenAI = None,
        temperature: float = None,
        max_tokens: int = None,
        custom_prompt: str = "",
        vector_store: VectorStore = None,
        k: int = None,
        pdf_parser: AzureFormRecognizerClient = None,
        blob_client: AzureBlobStorageClient = None,
        enable_translation: bool = False,
        translator: AzureTranslatorClient = None):

        load_dotenv()
        openai.api_type = "azure"
        openai.api_base = os.getenv('OPENAI_API_BASE')
        openai.api_version = "2023-03-15-preview"
        openai.api_key = os.getenv("OPENAI_API_KEY")

        # Azure OpenAI settings
        self.api_base = openai.api_base
        self.api_version = openai.api_version
        self.index_name: str = "embeddings"
        self.model: str = os.getenv('OPENAI_EMBEDDINGS_ENGINE_DOC', "text-embedding-ada-002")
        self.deployment_name: str = os.getenv("OPENAI_ENGINE", os.getenv("OPENAI_ENGINES", "text-davinci-003"))
        self.deployment_type: str = os.getenv("OPENAI_DEPLOYMENT_TYPE", "Text")
        self.temperature: float = float(os.getenv("OPENAI_TEMPERATURE", 0.7)) if temperature is None else temperature
        self.max_tokens: int = int(os.getenv("OPENAI_MAX_TOKENS", -1)) if max_tokens is None else max_tokens
        self.prompt = PROMPT if custom_prompt == '' else PromptTemplate(template=custom_prompt, input_variables=["summaries", "question"])
        self.vector_store_type = os.getenv("VECTOR_STORE_TYPE")

        # Azure Search settings
        if  self.vector_store_type == "AzureSearch":
            self.vector_store_address: str = os.getenv('AZURE_SEARCH_SERVICE_NAME')
            self.vector_store_password: str = os.getenv('AZURE_SEARCH_ADMIN_KEY')

        else:
            # Vector store settings
            self.vector_store_address: str = os.getenv('REDIS_ADDRESS', "localhost")
            self.vector_store_port: int= int(os.getenv('REDIS_PORT', 6379))
            self.vector_store_protocol: str = os.getenv("REDIS_PROTOCOL", "redis://")
            self.vector_store_password: str = os.getenv("REDIS_PASSWORD", None)

            if self.vector_store_password:
                self.vector_store_full_address = f"{self.vector_store_protocol}:{self.vector_store_password}@{self.vector_store_address}:{self.vector_store_port}"
            else:
                self.vector_store_full_address = f"{self.vector_store_protocol}{self.vector_store_address}:{self.vector_store_port}"

        self.chunk_size = int(os.getenv('CHUNK_SIZE', 500))
        self.chunk_overlap = int(os.getenv('CHUNK_OVERLAP', 100))
        self.document_loaders: BaseLoader = WebBaseLoader if document_loaders is None else document_loaders
        self.text_splitter: TextSplitter = TokenTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap) if text_splitter is None else text_splitter
        self.embeddings: OpenAIEmbeddings = OpenAIEmbeddings(model=self.model, chunk_size=1) if embeddings is None else embeddings
        if self.deployment_type == "Chat":
            self.llm: ChatOpenAI = ChatOpenAI(model_name=self.deployment_name, engine=self.deployment_name, temperature=self.temperature, max_tokens=self.max_tokens if self.max_tokens != -1 else None) if llm is None else llm
        else:
            self.llm: AzureOpenAI = NewAzureOpenAI(deployment_name=self.deployment_name, temperature=self.temperature, max_tokens=self.max_tokens) if llm is None else llm
        if self.vector_store_type == "AzureSearch":
            self.vector_store: VectorStore = AzureSearch(azure_cognitive_search_name=self.vector_store_address, azure_cognitive_search_key=self.vector_store_password, index_name=self.index_name, embedding_function=self.embeddings.embed_query) if vector_store is None else vector_store
        else:
            self.vector_store: RedisExtended = RedisExtended(redis_url=self.vector_store_full_address, index_name=self.index_name, embedding_function=self.embeddings.embed_query) if vector_store is None else vector_store   
        self.k : int = 3 if k is None else k

        self.pdf_parser : AzureFormRecognizerClient = AzureFormRecognizerClient() if pdf_parser is None else pdf_parser
        self.blob_client: AzureBlobStorageClient = AzureBlobStorageClient() if blob_client is None else blob_client
        self.enable_translation : bool = False if enable_translation is None else enable_translation
        self.translator : AzureTranslatorClient = AzureTranslatorClient() if translator is None else translator

        self.user_agent: UserAgent() = UserAgent()
        self.user_agent.random

        self.current_contextDict = {}
        self.current_sources = None
        self.current_answer = None
        langchain.verbose = True


    def add_embeddings_lc(self, source_url):
        retrycount = 0
        while True:
            try:
                documents = self.document_loaders(source_url).load()
                
                # Convert to UTF-8 encoding for non-ascii text
                for(document) in documents:
                    try:
                        if document.page_content.encode("iso-8859-1") == document.page_content.encode("latin-1"):
                            document.page_content = document.page_content.encode("iso-8859-1").decode("utf-8", errors="ignore")
                    except:
                        pass
                    
                docs = self.text_splitter.split_documents(documents)
                
                # Remove half non-ascii character from start/end of doc content (langchain TokenTextSplitter may split a non-ascii character in half)
                pattern = re.compile(r'[\x00-\x09\x0b\x0c\x0e-\x1f\x7f\u0080-\u00a0\u2000-\u3000\ufff0-\uffff]')  # do not remove \x0a (\n) nor \x0d (\r)
                for(doc) in docs:
                    doc.page_content = re.sub(pattern, '', doc.page_content)
                    if doc.page_content == '':
                        docs.remove(doc)
                
                keys = []
                for i, doc in enumerate(docs):
                    # Create a unique key for the document
                    source_url = source_url.split('?')[0]
                    filename = "/".join(source_url.split('/')[4:])
                    hash_key = hashlib.sha1(f"{source_url}_{i}".encode('utf-8')).hexdigest()
                    hash_key = f"doc:{self.index_name}:{hash_key}"
                    keys.append(hash_key)
                    doc.metadata = {"source": f"[{source_url}]({source_url}_SAS_TOKEN_PLACEHOLDER_)" , "chunk": i, "key": hash_key, "filename": filename}
                if self.vector_store_type == 'AzureSearch':
                    self.vector_store.add_documents(documents=docs, keys=keys)
                else:
                    self.vector_store.add_documents(documents=docs, redis_url=self.vector_store_full_address,  index_name=self.index_name, keys=keys)
                return

            except Exception as e:
                logging.error(f"Error adding embeddings for {source_url}: {e}")
                if retrycount > 3:
                    raise e
                else:
                    print(f"Retrying adding embeddings for {source_url}")
                    retrycount += 1

        

    def convert_file_and_add_embeddings(self, source_url, filename, enable_translation=False):
        # Extract the text from the file
        text = self.pdf_parser.analyze_read(source_url)
        # Translate if requested
        converted_text = list(map(lambda x: self.translator.translate(x), text)) if self.enable_translation else text

        # Remove half non-ascii character from start/end of doc content (langchain TokenTextSplitter may split a non-ascii character in half)
        pattern = re.compile(r'[\x00-\x09\x0b\x0c\x0e-\x1f\x7f\u0080-\u00a0\u2000-\u3000\ufff0-\uffff]')  # do not remove \x0a (\n) nor \x0d (\r)
        converted_text = re.sub(pattern, '', "\n".join(converted_text))

        # Upload the text to Azure Blob Storage
        converted_filename = f"converted/{filename}.txt"
        source_url = self.blob_client.upload_file(converted_text, f"converted/{filename}.txt", content_type='text/plain; charset=utf-8')

        print(f"Converted file uploaded to {source_url} with filename {filename}")
        # Update the metadata to indicate that the file has been converted
        self.blob_client.upsert_blob_metadata(filename, {"converted": "true"})

        self.add_embeddings_lc(source_url=source_url)

        return converted_filename

    def get_all_documents(self, k: int = None):
        result = self.vector_store.similarity_search(query="*", k= k if k else self.k)
        dataFrame = pd.DataFrame(list(map(lambda x: {
                'key': x.metadata['key'],
                'filename': x.metadata['filename'],
                'source': urllib.parse.unquote(x.metadata['source']), 
                'content': x.page_content, 
                'metadata' : x.metadata,
                }, result)))
        if dataFrame.empty is False:
            dataFrame = dataFrame.sort_values(by='filename')
        return dataFrame

    def get_semantic_answer_lang_chain(self, question, chat_history):
        question_generator = LLMChain(llm=self.llm, prompt=CONDENSE_QUESTION_PROMPT, verbose=True)
        doc_chain = load_qa_with_sources_chain(self.llm, chain_type="stuff", verbose=True, prompt=self.prompt)
        chain = ConversationalRetrievalChain(
            retriever=self.vector_store.as_retriever(),
            question_generator=question_generator,
            combine_docs_chain=doc_chain,
            return_source_documents=True,
            verbose=True,
            # top_k_docs_for_context= self.k
        )
        result = chain({"question": question, "chat_history": chat_history})
        sources = "\n".join(set(map(lambda x: x.metadata["source"], result['source_documents'])))

        container_sas = self.blob_client.get_container_sas()

        contextDict ={}
        for res in result['source_documents']:
            source_key = self.filter_sourcesLinks(res.metadata['source'].replace('_SAS_TOKEN_PLACEHOLDER_', container_sas)).replace('\n', '').replace(' ', '')
            if source_key not in contextDict:
                contextDict[source_key] = []
            myPageContent = self.clean_encoding(res.page_content)
            contextDict[source_key].append(myPageContent)
        
        result['answer'] = result['answer'].split('SOURCES:')[0].split('Sources:')[0].split('SOURCE:')[0].split('Source:')[0]
        result['answer'] = self.clean_encoding(result['answer'])
        sources = sources.replace('_SAS_TOKEN_PLACEHOLDER_', container_sas)
        sources = self.filter_sourcesLinks(sources)

        self.current_contextDict = contextDict
        self.current_sources = sources
        self.current_answer = result['answer']

        return result['answer'], contextDict, sources


    def get_general_operation_lang_chain(self, question, chat_history):

        lfTool = LifeKnowledgeSearchTool(config=LifeKnowledgeSearchConfig(chat_history=chat_history, get_semantic_answer_lang_chain_func=self.get_semantic_answer_lang_chain))
        tools = [
            MansionPriceTool(),
            lfTool,
            IotDeviceControlTool(),
            # Tool(
            #     name="生活の知恵機能",
            #     func=semantic_run,
            #     description="ユーザの問い合わせが生活の知恵が必要な場合は、生活の知恵ナレッジから返答することができます。"
            # ),
            # Tool(
            #     name="建物の価格機能",
            #     func=semantic_run,
            #     description="ユーザが引っ越しや新しい家を探している場合は、この機能をりようすることができます"
            # ), 
            # Tool(
            #     name="IoTデバイス制御",
            #     func=iot_device_run,
            #     description="ユーザが自宅のデバイスを制御したい際に利用することができます。"
            # ),
        ]

        agent = initialize_agent(
            tools,
            llm=self.llm,
            agent=AgentType.OPENAI_FUNCTIONS,
            verbose=True,
            return_intermediate_steps=True)

        result = agent({"input":question})


        return question, result['output'], self.current_contextDict, self.current_sources


    def get_embeddings_model(self):
        OPENAI_EMBEDDINGS_ENGINE_DOC = os.getenv('OPENAI_EMEBDDINGS_ENGINE', os.getenv('OPENAI_EMBEDDINGS_ENGINE_DOC', 'text-embedding-ada-002'))  
        OPENAI_EMBEDDINGS_ENGINE_QUERY = os.getenv('OPENAI_EMEBDDINGS_ENGINE', os.getenv('OPENAI_EMBEDDINGS_ENGINE_QUERY', 'text-embedding-ada-002'))
        return {
            "doc": OPENAI_EMBEDDINGS_ENGINE_DOC,
            "query": OPENAI_EMBEDDINGS_ENGINE_QUERY
        }

    def get_completion(self, prompt, **kwargs):
        if self.deployment_type == 'Chat':
            return self.llm([HumanMessage(content=prompt)]).content
        else:
            return self.llm(prompt)

    # remove paths from sources to only keep the filename
    def filter_sourcesLinks(self, sources):
        # use regex to replace all occurences of '[anypath/anypath/somefilename.xxx](the_link)' to '[somefilename](thelink)' in sources
        pattern = r'\[[^\]]*?/([^/\]]*?)\]'

        match = re.search(pattern, sources)
        while match:
            withoutExtensions = match.group(1).split('.')[0] # remove any extension to the name of the source document
            sources = sources[:match.start()] + f'[{withoutExtensions}]' + sources[match.end():]
            match = re.search(pattern, sources)
        
        sources = '  \n ' + sources.replace('\n', '  \n ') # add a carriage return after each source

        return sources

    def extract_followupquestions(self, answer):
        followupTag = answer.find('Follow-up Questions')
        followupQuestions = answer.find('<<')

        # take min of followupTag and folloupQuestions if not -1 to avoid taking the followup questions if there is no followupTag
        followupTag = min(followupTag, followupQuestions) if followupTag != -1 and followupQuestions != -1 else max(followupTag, followupQuestions)
        answer_without_followupquestions = answer[:followupTag] if followupTag != -1 else answer
        followup_questions = answer[followupTag:].strip() if followupTag != -1 else ''

        # Extract the followup questions as a list
        pattern = r'\<\<(.*?)\>\>'
        match = re.search(pattern, followup_questions)
        followup_questions_list = []
        while match:
            followup_questions_list.append(followup_questions[match.start()+2:match.end()-2])
            followup_questions = followup_questions[match.end():]
            match = re.search(pattern, followup_questions)
        
        if followup_questions_list != '':
            # Extract follow up question 
            pattern = r'\d. (.*)'
            match = re.search(pattern, followup_questions)
            while match:
                followup_questions_list.append(followup_questions[match.start()+3:match.end()])
                followup_questions = followup_questions[match.end():]
                match = re.search(pattern, followup_questions)

        if followup_questions_list != '':
            pattern = r'Follow-up Question: (.*)'
            match = re.search(pattern, followup_questions)
            while match:
                followup_questions_list.append(followup_questions[match.start()+19:match.end()])
                followup_questions = followup_questions[match.end():]
                match = re.search(pattern, followup_questions)
        
        # Special case when 'Follow-up questions:' appears in the answer after the <<
        followupTag = answer_without_followupquestions.lower().find('follow-up questions')
        if followupTag != -1:
            answer_without_followupquestions = answer_without_followupquestions[:followupTag]
        followupTag = answer_without_followupquestions.lower().find('follow up questions')  # LLM can make variations...
        if followupTag != -1:
            answer_without_followupquestions = answer_without_followupquestions[:followupTag]

        return answer_without_followupquestions, followup_questions_list

    # insert citations in the answer - find filenames in the answer maching sources from the filenamelist and replace them with '${(id+1)}'
    def insert_citations_in_answer(self, answer, filenameList):

        filenameList_lowered = [x.lower() for x in filenameList]    # LLM can make case mitakes in returing the filename of the source

        matched_sources = []
        pattern = r'\[\[(.*?)\]\]'
        match = re.search(pattern, answer)
        while match:
            filename = match.group(1).split('.')[0] # remove any extension to the name of the source document
            if filename in filenameList:
                if filename not in matched_sources:
                    matched_sources.append(filename.lower())
                filenameIndex = filenameList.index(filename) + 1
                answer = answer[:match.start()] + '$^{' + f'{filenameIndex}' + '}$' + answer[match.end():]
            else:
                answer = answer[:match.start()] + '$^{' + f'{filename.lower()}' + '}$' + answer[match.end():]
            match = re.search(pattern, answer)

        # When page is reloaded search for references already added to the answer (e.g. '${(id+1)}')
        for id, filename in enumerate(filenameList_lowered):
            reference = '$^{' + f'{id+1}' + '}$'
            if reference in answer and not filename in matched_sources:
                matched_sources.append(filename)

        return answer, matched_sources, filenameList_lowered

    def get_links_filenames(self, answer, sources):
        
        if sources != None:
            split_sources = sources.split('  \n ') # soures are expected to be of format '  \n  [filename1.ext](sourcelink1)  \n [filename2.ext](sourcelink2)  \n  [filename3.ext](sourcelink3)  \n '
        else:
            split_sources = []
            
        srcList = []
        linkList = []
        filenameList = []
        for src in split_sources:
            if src != '':
                srcList.append(src)
                link = src[1:].split('(')[1][:-1].split(')')[0] # get the link
                linkList.append(link)
                filename = src[1:].split(']')[0] # retrieve the source filename.
                source_url = link.split('?')[0]
                answer = answer.replace(source_url, filename)  # if LLM added a path to the filename, remove it from the answer
                filenameList.append(filename)

        answer, matchedSourcesList, filenameList = self.insert_citations_in_answer(answer, filenameList) # Add (1), (2), (3) to the answer to indicate the source of the answer

        return answer, srcList, matchedSourcesList, linkList, filenameList

    def clean_encoding(self, text):
        try:
            encoding = 'ISO-8859-1'
            encodedtext = text.encode(encoding)
            encodedtext = encodedtext.decode('utf-8')
        except Exception as e:
            encodedtext = text
        return encodedtext

langchain.agents.initialize_agent langchain.chains.llm.LLMChain langchain.chat_models.ChatOpenAI langchain.schema.HumanMessage langchain.text_splitter.TokenTextSplitter langchain.chains.qa_with_sources.load_qa_with_sources_chain langchain.prompts.PromptTemplate langchain.embeddings.openai.OpenAIEmbeddings
[((2224, 2237), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (2235, 2237), False, 'from dotenv import load_dotenv\n'), ((2298, 2326), 'os.getenv', 'os.getenv', (['"""OPENAI_API_BASE"""'], {}), "('OPENAI_API_BASE')\n", (2307, 2326), False, 'import os\n'), ((2402, 2429), 'os.getenv', 'os.getenv', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (2411, 2429), False, 'import os\n'), ((2619, 2686), 'os.getenv', 'os.getenv', (['"""OPENAI_EMBEDDINGS_ENGINE_DOC"""', '"""text-embedding-ada-002"""'], {}), "('OPENAI_EMBEDDINGS_ENGINE_DOC', 'text-embedding-ada-002')\n", (2628, 2686), False, 'import os\n'), ((2835, 2878), 'os.getenv', 'os.getenv', (['"""OPENAI_DEPLOYMENT_TYPE"""', '"""Text"""'], {}), "('OPENAI_DEPLOYMENT_TYPE', 'Text')\n", (2844, 2878), False, 'import os\n'), ((3278, 3308), 'os.getenv', 'os.getenv', (['"""VECTOR_STORE_TYPE"""'], {}), "('VECTOR_STORE_TYPE')\n", (3287, 3308), False, 'import os\n'), ((6379, 6390), 'fake_useragent.UserAgent', 'UserAgent', ([], {}), '()\n', (6388, 6390), False, 'from fake_useragent import UserAgent\n'), ((6393, 6404), 'fake_useragent.UserAgent', 'UserAgent', ([], {}), '()\n', (6402, 6404), False, 'from fake_useragent import UserAgent\n'), ((9603, 9711), 're.compile', 're.compile', (['"""[\\\\x00-\\\\x09\\\\x0b\\\\x0c\\\\x0e-\\\\x1f\\\\x7f\\\\u0080-\\\\u00a0\\\\u2000-\\\\u3000\\\\ufff0-\\\\uffff]"""'], {}), "(\n    '[\\\\x00-\\\\x09\\\\x0b\\\\x0c\\\\x0e-\\\\x1f\\\\x7f\\\\u0080-\\\\u00a0\\\\u2000-\\\\u3000\\\\ufff0-\\\\uffff]'\n    )\n", (9613, 9711), False, 'import re\n'), ((11066, 11135), 'langchain.chains.llm.LLMChain', 'LLMChain', ([], {'llm': 'self.llm', 'prompt': 'CONDENSE_QUESTION_PROMPT', 'verbose': '(True)'}), '(llm=self.llm, prompt=CONDENSE_QUESTION_PROMPT, verbose=True)\n', (11074, 11135), False, 'from langchain.chains.llm import LLMChain\n'), ((11156, 11250), 'langchain.chains.qa_with_sources.load_qa_with_sources_chain', 'load_qa_with_sources_chain', (['self.llm'], {'chain_type': '"""stuff"""', 'verbose': '(True)', 'prompt': 'self.prompt'}), "(self.llm, chain_type='stuff', verbose=True,\n    prompt=self.prompt)\n", (11182, 11250), False, 'from langchain.chains.qa_with_sources import load_qa_with_sources_chain\n'), ((13693, 13814), 'langchain.agents.initialize_agent', 'initialize_agent', (['tools'], {'llm': 'self.llm', 'agent': 'AgentType.OPENAI_FUNCTIONS', 'verbose': '(True)', 'return_intermediate_steps': '(True)'}), '(tools, llm=self.llm, agent=AgentType.OPENAI_FUNCTIONS,\n    verbose=True, return_intermediate_steps=True)\n', (13709, 13814), False, 'from langchain.agents import initialize_agent, Tool, AgentType\n'), ((14982, 15009), 're.search', 're.search', (['pattern', 'sources'], {}), '(pattern, sources)\n', (14991, 15009), False, 'import re\n'), ((16149, 16187), 're.search', 're.search', (['pattern', 'followup_questions'], {}), '(pattern, followup_questions)\n', (16158, 16187), False, 'import re\n'), ((18363, 18389), 're.search', 're.search', (['pattern', 'answer'], {}), '(pattern, answer)\n', (18372, 18389), False, 'import re\n'), ((2750, 2797), 'os.getenv', 'os.getenv', (['"""OPENAI_ENGINES"""', '"""text-davinci-003"""'], {}), "('OPENAI_ENGINES', 'text-davinci-003')\n", (2759, 2797), False, 'import os\n'), ((3163, 3248), 'langchain.prompts.PromptTemplate', 'PromptTemplate', ([], {'template': 'custom_prompt', 'input_variables': "['summaries', 'question']"}), "(template=custom_prompt, input_variables=['summaries',\n    'question'])\n", (3177, 3248), False, 'from langchain.prompts import PromptTemplate\n'), ((3440, 3478), 'os.getenv', 'os.getenv', (['"""AZURE_SEARCH_SERVICE_NAME"""'], {}), "('AZURE_SEARCH_SERVICE_NAME')\n", (3449, 3478), False, 'import os\n'), ((3525, 3560), 'os.getenv', 'os.getenv', (['"""AZURE_SEARCH_ADMIN_KEY"""'], {}), "('AZURE_SEARCH_ADMIN_KEY')\n", (3534, 3560), False, 'import os\n'), ((3657, 3696), 'os.getenv', 'os.getenv', (['"""REDIS_ADDRESS"""', '"""localhost"""'], {}), "('REDIS_ADDRESS', 'localhost')\n", (3666, 3696), False, 'import os\n'), ((3819, 3858), 'os.getenv', 'os.getenv', (['"""REDIS_PROTOCOL"""', '"""redis://"""'], {}), "('REDIS_PROTOCOL', 'redis://')\n", (3828, 3858), False, 'import os\n'), ((3905, 3938), 'os.getenv', 'os.getenv', (['"""REDIS_PASSWORD"""', 'None'], {}), "('REDIS_PASSWORD', None)\n", (3914, 3938), False, 'import os\n'), ((4328, 4356), 'os.getenv', 'os.getenv', (['"""CHUNK_SIZE"""', '(500)'], {}), "('CHUNK_SIZE', 500)\n", (4337, 4356), False, 'import os\n'), ((4391, 4422), 'os.getenv', 'os.getenv', (['"""CHUNK_OVERLAP"""', '(100)'], {}), "('CHUNK_OVERLAP', 100)\n", (4400, 4422), False, 'import os\n'), ((4575, 4654), 'langchain.text_splitter.TokenTextSplitter', 'TokenTextSplitter', ([], {'chunk_size': 'self.chunk_size', 'chunk_overlap': 'self.chunk_overlap'}), '(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)\n', (4592, 4654), False, 'from langchain.text_splitter import TokenTextSplitter, TextSplitter\n'), ((4743, 4791), 'langchain.embeddings.openai.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {'model': 'self.model', 'chunk_size': '(1)'}), '(model=self.model, chunk_size=1)\n', (4759, 4791), False, 'from langchain.embeddings.openai import OpenAIEmbeddings\n'), ((5958, 5985), 'utilities.formrecognizer.AzureFormRecognizerClient', 'AzureFormRecognizerClient', ([], {}), '()\n', (5983, 5985), False, 'from utilities.formrecognizer import AzureFormRecognizerClient\n'), ((6075, 6099), 'utilities.azureblobstorage.AzureBlobStorageClient', 'AzureBlobStorageClient', ([], {}), '()\n', (6097, 6099), False, 'from utilities.azureblobstorage import AzureBlobStorageClient\n'), ((6291, 6314), 'utilities.translator.AzureTranslatorClient', 'AzureTranslatorClient', ([], {}), '()\n', (6312, 6314), False, 'from utilities.translator import AzureTranslatorClient\n'), ((13045, 13063), 'utilities.tools.MansionPriceTool', 'MansionPriceTool', ([], {}), '()\n', (13061, 13063), False, 'from utilities.tools import LifeKnowledgeSearchTool, IotDeviceControlTool, MansionPriceTool, LifeKnowledgeSearchConfig\n'), ((13097, 13119), 'utilities.tools.IotDeviceControlTool', 'IotDeviceControlTool', ([], {}), '()\n', (13117, 13119), False, 'from utilities.tools import LifeKnowledgeSearchTool, IotDeviceControlTool, MansionPriceTool, LifeKnowledgeSearchConfig\n'), ((14123, 14190), 'os.getenv', 'os.getenv', (['"""OPENAI_EMBEDDINGS_ENGINE_DOC"""', '"""text-embedding-ada-002"""'], {}), "('OPENAI_EMBEDDINGS_ENGINE_DOC', 'text-embedding-ada-002')\n", (14132, 14190), False, 'import os\n'), ((14273, 14342), 'os.getenv', 'os.getenv', (['"""OPENAI_EMBEDDINGS_ENGINE_QUERY"""', '"""text-embedding-ada-002"""'], {}), "('OPENAI_EMBEDDINGS_ENGINE_QUERY', 'text-embedding-ada-002')\n", (14282, 14342), False, 'import os\n'), ((15267, 15294), 're.search', 're.search', (['pattern', 'sources'], {}), '(pattern, sources)\n', (15276, 15294), False, 'import re\n'), ((16426, 16464), 're.search', 're.search', (['pattern', 'followup_questions'], {}), '(pattern, followup_questions)\n', (16435, 16464), False, 'import re\n'), ((16612, 16650), 're.search', 're.search', (['pattern', 'followup_questions'], {}), '(pattern, followup_questions)\n', (16621, 16650), False, 'import re\n'), ((17018, 17056), 're.search', 're.search', (['pattern', 'followup_questions'], {}), '(pattern, followup_questions)\n', (17027, 17056), False, 'import re\n'), ((18995, 19021), 're.search', 're.search', (['pattern', 'answer'], {}), '(pattern, answer)\n', (19004, 19021), False, 'import re\n'), ((2919, 2955), 'os.getenv', 'os.getenv', (['"""OPENAI_TEMPERATURE"""', '(0.7)'], {}), "('OPENAI_TEMPERATURE', 0.7)\n", (2928, 2955), False, 'import os\n'), ((3032, 3066), 'os.getenv', 'os.getenv', (['"""OPENAI_MAX_TOKENS"""', '(-1)'], {}), "('OPENAI_MAX_TOKENS', -1)\n", (3041, 3066), False, 'import os\n'), ((3742, 3771), 'os.getenv', 'os.getenv', (['"""REDIS_PORT"""', '(6379)'], {}), "('REDIS_PORT', 6379)\n", (3751, 3771), False, 'import os\n'), ((4908, 5082), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'model_name': 'self.deployment_name', 'engine': 'self.deployment_name', 'temperature': 'self.temperature', 'max_tokens': '(self.max_tokens if self.max_tokens != -1 else None)'}), '(model_name=self.deployment_name, engine=self.deployment_name,\n    temperature=self.temperature, max_tokens=self.max_tokens if self.\n    max_tokens != -1 else None)\n', (4918, 5082), False, 'from langchain.chat_models import ChatOpenAI\n'), ((5148, 5263), 'utilities.NewAzureOpenAI.NewAzureOpenAI', 'NewAzureOpenAI', ([], {'deployment_name': 'self.deployment_name', 'temperature': 'self.temperature', 'max_tokens': 'self.max_tokens'}), '(deployment_name=self.deployment_name, temperature=self.\n    temperature, max_tokens=self.max_tokens)\n', (5162, 5263), False, 'from utilities.NewAzureOpenAI import NewAzureOpenAI\n'), ((5380, 5586), 'utilities.azuresearch.AzureSearch', 'AzureSearch', ([], {'azure_cognitive_search_name': 'self.vector_store_address', 'azure_cognitive_search_key': 'self.vector_store_password', 'index_name': 'self.index_name', 'embedding_function': 'self.embeddings.embed_query'}), '(azure_cognitive_search_name=self.vector_store_address,\n    azure_cognitive_search_key=self.vector_store_password, index_name=self.\n    index_name, embedding_function=self.embeddings.embed_query)\n', (5391, 5586), False, 'from utilities.azuresearch import AzureSearch\n'), ((5681, 5817), 'utilities.redis.RedisExtended', 'RedisExtended', ([], {'redis_url': 'self.vector_store_full_address', 'index_name': 'self.index_name', 'embedding_function': 'self.embeddings.embed_query'}), '(redis_url=self.vector_store_full_address, index_name=self.\n    index_name, embedding_function=self.embeddings.embed_query)\n', (5694, 5817), False, 'from utilities.redis import RedisExtended\n'), ((7486, 7594), 're.compile', 're.compile', (['"""[\\\\x00-\\\\x09\\\\x0b\\\\x0c\\\\x0e-\\\\x1f\\\\x7f\\\\u0080-\\\\u00a0\\\\u2000-\\\\u3000\\\\ufff0-\\\\uffff]"""'], {}), "(\n    '[\\\\x00-\\\\x09\\\\x0b\\\\x0c\\\\x0e-\\\\x1f\\\\x7f\\\\u0080-\\\\u00a0\\\\u2000-\\\\u3000\\\\ufff0-\\\\uffff]'\n    )\n", (7496, 7594), False, 'import re\n'), ((12888, 13017), 'utilities.tools.LifeKnowledgeSearchConfig', 'LifeKnowledgeSearchConfig', ([], {'chat_history': 'chat_history', 'get_semantic_answer_lang_chain_func': 'self.get_semantic_answer_lang_chain'}), '(chat_history=chat_history,\n    get_semantic_answer_lang_chain_func=self.get_semantic_answer_lang_chain)\n', (12913, 13017), False, 'from utilities.tools import LifeKnowledgeSearchTool, IotDeviceControlTool, MansionPriceTool, LifeKnowledgeSearchConfig\n'), ((16866, 16904), 're.search', 're.search', (['pattern', 'followup_questions'], {}), '(pattern, followup_questions)\n', (16875, 16904), False, 'import re\n'), ((17273, 17311), 're.search', 're.search', (['pattern', 'followup_questions'], {}), '(pattern, followup_questions)\n', (17282, 17311), False, 'import re\n'), ((7687, 7724), 're.sub', 're.sub', (['pattern', '""""""', 'doc.page_content'], {}), "(pattern, '', doc.page_content)\n", (7693, 7724), False, 'import re\n'), ((8829, 8892), 'logging.error', 'logging.error', (['f"""Error adding embeddings for {source_url}: {e}"""'], {}), "(f'Error adding embeddings for {source_url}: {e}')\n", (8842, 8892), False, 'import logging\n'), ((14593, 14621), 'langchain.schema.HumanMessage', 'HumanMessage', ([], {'content': 'prompt'}), '(content=prompt)\n', (14605, 14621), False, 'from langchain.schema import AIMessage, HumanMessage, SystemMessage\n'), ((10684, 10726), 'urllib.parse.unquote', 'urllib.parse.unquote', (["x.metadata['source']"], {}), "(x.metadata['source'])\n", (10704, 10726), False, 'import urllib\n')]