import logging, json, os
from Utilities.envVars import *
from Utilities.envVars import *
# Import required libraries
from Utilities.cogSearchVsRetriever import CognitiveSearchVsRetriever
from langchain.chains import RetrievalQA
from langchain import PromptTemplate
from Utilities.evaluator import indexDocs
import json
import time
import pandas as pd
from collections import namedtuple
from Utilities.evaluator import searchEvaluatorRunIdIndex
import uuid
import tempfile
from Utilities.azureBlob import getBlob, getFullPath
from langchain.document_loaders import PDFMinerLoader, UnstructuredFileLoader
from Utilities.evaluator import createEvaluatorResultIndex, searchEvaluatorRunIdIndex
from langchain.chat_models import AzureChatOpenAI, ChatOpenAI
from langchain.evaluation.qa import QAEvalChain
from Utilities.evaluator import searchEvaluatorRunIndex, createEvaluatorRunIndex, getEvaluatorResult

RunDocs = namedtuple('RunDoc', ['evalatorQaData', 'totalQuestions', 'promptStyle', 'documentId', 
                                        'splitMethods', 'chunkSizes', 'overlaps',
                                        'retrieverType', 'reEvaluate', 'topK', 'model', 'fileName',
                                        'embeddingModelType', 'temperature', 'tokenLength'])

def getPrompts():
    template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. Use three sentences maximum. Keep the answer as concise as possible.
    {context}
    Question: {question}
    Helpful Answer:"""

    QaChainPrompt = PromptTemplate(input_variables=["context", "question"],template=template,)

    template = """You are a teacher grading a quiz. 
    You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect.

    Example Format:
    QUESTION: question here
    STUDENT ANSWER: student's answer here
    TRUE ANSWER: true answer here
    GRADE: Correct or Incorrect here

    Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student answers that there is no specific information provided in the context, then the answer is Incorrect. Begin! 

    QUESTION: {query}
    STUDENT ANSWER: {result}
    TRUE ANSWER: {answer}
    GRADE:"""

    promptStyleFast = PromptTemplate(input_variables=["query", "result", "answer"], template=template)

    template = """You are a teacher grading a quiz. 
    You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect.
    You are also asked to identify potential sources of bias in the question and in the true answer.

    Example Format:
    QUESTION: question here
    STUDENT ANSWER: student's answer here
    TRUE ANSWER: true answer here
    GRADE: Correct or Incorrect here

    Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student answers that there is no specific information provided in the context, then the answer is Incorrect. Begin! 

    QUESTION: {query}
    STUDENT ANSWER: {result}
    TRUE ANSWER: {answer}
    GRADE:

    Your response should be as follows:

    GRADE: (Correct or Incorrect)
    (line break)
    JUSTIFICATION: (Without mentioning the student/teacher framing of this prompt, explain why the STUDENT ANSWER is Correct or Incorrect, identify potential sources of bias in the QUESTION, and identify potential sources of bias in the TRUE ANSWER. Use one or two sentences maximum. Keep the answer as concise as possible.)
    """

    promptStyleBias = PromptTemplate(input_variables=["query", "result", "answer"], template=template)

    template = """You are assessing a submitted student answer to a question relative to the true answer based on the provided criteria: 
    
        ***
        QUESTION: {query}
        ***
        STUDENT ANSWER: {result}
        ***
        TRUE ANSWER: {answer}
        ***
        Criteria: 
        relevance:  Is the submission referring to a real quote from the text?"
        conciseness:  Is the answer concise and to the point?"
        correct: Is the answer correct?"
        ***
        Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print "Correct" or "Incorrect" (without quotes or punctuation) on its own line corresponding to the correct answer.
        Reasoning:
    """

    promptStyleGrading = PromptTemplate(input_variables=["query", "result", "answer"], template=template)

    template = """You are a teacher grading a quiz. 
    You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect.

    Example Format:
    QUESTION: question here
    STUDENT ANSWER: student's answer here
    TRUE ANSWER: true answer here
    GRADE: Correct or Incorrect here

    Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student answers that there is no specific information provided in the context, then the answer is Incorrect. Begin! 

    QUESTION: {query}
    STUDENT ANSWER: {result}
    TRUE ANSWER: {answer}
    GRADE:

    Your response should be as follows:

    GRADE: (Correct or Incorrect)
    (line break)
    JUSTIFICATION: (Without mentioning the student/teacher framing of this prompt, explain why the STUDENT ANSWER is Correct or Incorrect. Use one or two sentences maximum. Keep the answer as concise as possible.)
    """

    promptStyleDefault = PromptTemplate(input_variables=["query", "result", "answer"], template=template)

    template = """ 
        Given the question: \n
        {query}
        Here are some documents retrieved in response to the question: \n
        {result}
        And here is the answer to the question: \n 
        {answer}
        Criteria: 
        relevance: Are the retrieved documents relevant to the question and do they support the answer?"
        Do the retrieved documents meet the criterion? Print "Correct" (without quotes or punctuation) if the retrieved context are relevant or "Incorrect" if not (without quotes or punctuation) on its own line. """

    gradeDocsPromptFast = PromptTemplate(input_variables=["query", "result", "answer"], template=template)

    template = """ 
        Given the question: \n
        {query}
        Here are some documents retrieved in response to the question: \n
        {result}
        And here is the answer to the question: \n 
        {answer}
        Criteria: 
        relevance: Are the retrieved documents relevant to the question and do they support the answer?"

        Your response should be as follows:

        GRADE: (Correct or Incorrect, depending if the retrieved documents meet the criterion)
        (line break)
        JUSTIFICATION: (Write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Use one or two sentences maximum. Keep the answer as concise as possible.)
        """

    gradeDocsPromptDefault = PromptTemplate(input_variables=["query", "result", "answer"], template=template)

    return QaChainPrompt, promptStyleFast, promptStyleBias, promptStyleGrading, promptStyleDefault, gradeDocsPromptFast, gradeDocsPromptDefault

def gradeModelAnswer(llm, predictedDataSet, predictions, promptStyle, promptStyleFast, promptStyleBias, promptStyleGrading, promptStyleDefault):

    if promptStyle == "Fast":
        prompt = promptStyleFast
    elif promptStyle == "Descriptive w/ bias check":
        prompt = promptStyleBias
    elif promptStyle == "OpenAI grading prompt":
        prompt = promptStyleGrading
    else:
        prompt = promptStyleDefault

    # Note: GPT-4 grader is advised by OAI 
    evalChain = QAEvalChain.from_llm(llm=llm,
                                      prompt=prompt)
    gradedOutputs = evalChain.evaluate(predictedDataSet,
                                         predictions,
                                         question_key="question",
                                         prediction_key="result")
    return gradedOutputs

def gradeModelRetrieval(llm, getDataSet, predictions, gradeDocsPrompt, gradeDocsPromptFast, gradeDocsPromptDefault):

    if gradeDocsPrompt == "Fast":
        prompt = gradeDocsPromptFast
    else:
        prompt = gradeDocsPromptDefault

    # Note: GPT-4 grader is advised by OAI
    evalChain = QAEvalChain.from_llm(llm=llm,prompt=prompt)
    gradedOutputs = evalChain.evaluate(getDataSet,
                                         predictions,
                                         question_key="question",
                                         prediction_key="result")
    return gradedOutputs

def blobLoad(blobConnectionString, blobContainer, blobName):
    readBytes  = getBlob(blobConnectionString, blobContainer, blobName)
    downloadPath = os.path.join(tempfile.gettempdir(), blobName)
    os.makedirs(os.path.dirname(tempfile.gettempdir()), exist_ok=True)
    try:
        with open(downloadPath, "wb") as file:
            file.write(readBytes)
    except Exception as e:
        logging.error(e)

    logging.info("File created " + downloadPath)
    if (blobName.endswith(".pdf")):
        loader = PDFMinerLoader(downloadPath)

    rawDocs = loader.load()

    fullPath = getFullPath(blobConnectionString, blobContainer, blobName)
    for doc in rawDocs:
        doc.metadata['source'] = fullPath
    return rawDocs

def runEvaluator(llm, evaluatorQaData, totalQuestions, chain, retriever, promptStyle, 
                 promptStyleFast, promptStyleBias, promptStyleGrading, promptStyleDefault,
                 gradeDocsPromptFast, gradeDocsPromptDefault) -> list:
    d = pd.DataFrame(columns=['question', 'answer', 'predictedAnswer', 'answerScore', 'retrievalScore', 'latency'])
    for i in range(int(totalQuestions)):
        predictions = []
        retrievedDocs = []
        gtDataSet = []
        latency = []
        currentDataSet = evaluatorQaData[i]
    
        try:
            startTime = time.time()
            predictions.append(chain({"query": currentDataSet["question"]}, return_only_outputs=True))
            gtDataSet.append(currentDataSet)
            endTime = time.time()
            elapsedTime = endTime - startTime
            latency.append(elapsedTime)
        except:
            predictions.append({'result': 'Error in prediction'})
            print("Error in prediction")

        # Extract text from retrieved docs
        retrievedDocText = ""
        docs = retriever.get_relevant_documents(currentDataSet["question"])
        for i, doc in enumerate(docs):
            retrievedDocText += "Doc %s: " % str(i+1) + \
                doc.page_content + " "

        # Log
        retrieved = {"question": currentDataSet["question"],
                    "answer": currentDataSet["answer"], "result": retrievedDocText}
        retrievedDocs.append(retrieved)

        # Grade
        gradedAnswer = gradeModelAnswer(llm, gtDataSet, predictions, promptStyle, promptStyleFast, promptStyleBias, promptStyleGrading, promptStyleDefault)
        gradedRetrieval = gradeModelRetrieval(llm, gtDataSet, retrievedDocs, promptStyle, gradeDocsPromptFast, gradeDocsPromptDefault)

        # Assemble output
        # Summary statistics
        dfOutput = {'question': currentDataSet['question'], 'answer': currentDataSet['answer'],
                    'predictedAnswer': predictions[0]['result'], 'answerScore': [{'score': 1 if "Incorrect" not in text else 0,
                                'justification': text} for text in [g['text'] for g in gradedAnswer]], 
                                'retrievalScore': [{'score': 1 if "Incorrect" not in text else 0,
                                'justification': text} for text in [g['text'] for g in gradedRetrieval]],
                    'latency': latency}
        
        #yield dfOutput

        # Add to dataframe
        d = pd.concat([d, pd.DataFrame(dfOutput)], axis=0)

    d_dict = d.to_dict('records')
    return d_dict

def main(runDocs: RunDocs) -> str:
    evaluatorQaData,totalQuestions,promptStyle,documentId,splitMethods,chunkSizes,overlaps,retrieverType,reEvaluate,topK,model,fileName, embeddingModelType, temperature, tokenLength = runDocs

    evaluatorDataIndexName = "evaluatordata"
    evaluatorRunIndexName = "evaluatorrun"
    evaluatorRunResultIndexName = "evaluatorrunresult"

    qaChainPrompt, promptStyleFast, promptStyleBias, promptStyleGrading, promptStyleDefault, gradeDocsPromptFast, gradeDocsPromptDefault = getPrompts()

    logging.info("Python HTTP trigger function processed a request.")
       
    if (embeddingModelType == 'azureopenai'):
            llm = AzureChatOpenAI(
                        azure_endpoint=OpenAiEndPoint,
                        api_version=OpenAiVersion,
                        azure_deployment=OpenAiChat,
                        temperature=temperature,
                        api_key=OpenAiKey,
                        max_tokens=tokenLength)
            logging.info("LLM Setup done")
    elif embeddingModelType == "openai":
            llm = ChatOpenAI(temperature=temperature,
                api_key=OpenAiApiKey,
                model_name="gpt-3.5-turbo",
                max_tokens=tokenLength)

    # Select retriever
    createEvaluatorResultIndex(SearchService, SearchKey, evaluatorRunResultIndexName)
    # Check if we already have runId for this document
    r = searchEvaluatorRunIdIndex(SearchService, SearchKey, evaluatorRunResultIndexName, documentId)
    if r.get_count() == 0:
        runId = str(uuid.uuid4())
    else:
        for run in r:
            runId = run['runId']
            break
    for splitMethod in splitMethods:
        for chunkSize in chunkSizes:
            for overlap in overlaps:
                # Verify if we have created the Run ID
                r = searchEvaluatorRunIndex(SearchService, SearchKey, evaluatorRunResultIndexName, documentId, retrieverType, 
                                        promptStyle, splitMethod, chunkSize, overlap)
                if r.get_count() == 0 or reEvaluate:
                    # Create the Run ID
                    print("Processing: ", documentId, retrieverType, promptStyle, splitMethod, chunkSize, overlap)
                    runIdData = []
                    subRunId = str(uuid.uuid4())
                
                    retriever = CognitiveSearchVsRetriever(contentKey="contentVector",
                                serviceName=SearchService,
                                apiKey=SearchKey,
                                indexName=evaluatorDataIndexName,
                                topK=topK,
                                splitMethod = splitMethod,
                                model = model,
                                chunkSize = chunkSize,
                                overlap = overlap,
                                openAiEndPoint = OpenAiEndPoint,
                                openAiKey = OpenAiKey,
                                openAiVersion = OpenAiVersion,
                                openAiApiKey = OpenAiApiKey,
                                documentId = documentId,
                                openAiEmbedding=OpenAiEmbedding,
                                returnFields=["id", "content", "sourceFile", "splitMethod", "chunkSize", "overlap", "model", "modelType", "documentId"]
                                )
                    vectorStoreChain = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, 
                                                    chain_type_kwargs={"prompt": qaChainPrompt})
                    runEvaluations = runEvaluator(llm, evaluatorQaData, totalQuestions, vectorStoreChain, retriever, promptStyle, 
                                                  promptStyleFast, promptStyleBias, promptStyleGrading, promptStyleDefault,
                                                    gradeDocsPromptFast, gradeDocsPromptDefault)
                    #yield runEvaluations
                    
                    runEvaluationData = []
                    for runEvaluation in runEvaluations:
                            runEvaluationData.append({
                                "id": str(uuid.uuid4()),
                                "runId": runId,
                                "subRunId": subRunId,
                                "documentId": documentId,
                                "retrieverType": retrieverType,
                                "promptStyle": promptStyle,
                                "splitMethod": splitMethod,
                                "chunkSize": chunkSize,
                                "overlap": overlap,
                                "question": runEvaluation['question'],
                                "answer": runEvaluation['answer'],
                                "predictedAnswer": runEvaluation['predictedAnswer'],
                                "answerScore": json.dumps(runEvaluation['answerScore']),
                                "retrievalScore": json.dumps(runEvaluation['retrievalScore']),
                                "latency": str(runEvaluation['latency']),
                            })
                    indexDocs(SearchService, SearchKey, evaluatorRunResultIndexName, runEvaluationData)
                    
    return "Success"
langchain.evaluation.qa.QAEvalChain.from_llm langchain.chains.RetrievalQA.from_chain_type langchain.chat_models.ChatOpenAI langchain.document_loaders.PDFMinerLoader langchain.chat_models.AzureChatOpenAI langchain.PromptTemplate
[((911, 1164), 'collections.namedtuple', 'namedtuple', (['"""RunDoc"""', "['evalatorQaData', 'totalQuestions', 'promptStyle', 'documentId',\n    'splitMethods', 'chunkSizes', 'overlaps', 'retrieverType', 'reEvaluate',\n    'topK', 'model', 'fileName', 'embeddingModelType', 'temperature',\n    'tokenLength']"], {}), "('RunDoc', ['evalatorQaData', 'totalQuestions', 'promptStyle',\n    'documentId', 'splitMethods', 'chunkSizes', 'overlaps', 'retrieverType',\n    'reEvaluate', 'topK', 'model', 'fileName', 'embeddingModelType',\n    'temperature', 'tokenLength'])\n", (921, 1164), False, 'from collections import namedtuple\n'), ((1626, 1700), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['context', 'question']", 'template': 'template'}), "(input_variables=['context', 'question'], template=template)\n", (1640, 1700), False, 'from langchain import PromptTemplate\n'), ((2601, 2686), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['query', 'result', 'answer']", 'template': 'template'}), "(input_variables=['query', 'result', 'answer'], template=template\n    )\n", (2615, 2686), False, 'from langchain import PromptTemplate\n'), ((4106, 4191), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['query', 'result', 'answer']", 'template': 'template'}), "(input_variables=['query', 'result', 'answer'], template=template\n    )\n", (4120, 4191), False, 'from langchain import PromptTemplate\n'), ((5082, 5167), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['query', 'result', 'answer']", 'template': 'template'}), "(input_variables=['query', 'result', 'answer'], template=template\n    )\n", (5096, 5167), False, 'from langchain import PromptTemplate\n'), ((6378, 6463), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['query', 'result', 'answer']", 'template': 'template'}), "(input_variables=['query', 'result', 'answer'], template=template\n    )\n", (6392, 6463), False, 'from langchain import PromptTemplate\n'), ((7054, 7139), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['query', 'result', 'answer']", 'template': 'template'}), "(input_variables=['query', 'result', 'answer'], template=template\n    )\n", (7068, 7139), False, 'from langchain import PromptTemplate\n'), ((7904, 7989), 'langchain.PromptTemplate', 'PromptTemplate', ([], {'input_variables': "['query', 'result', 'answer']", 'template': 'template'}), "(input_variables=['query', 'result', 'answer'], template=template\n    )\n", (7918, 7989), False, 'from langchain import PromptTemplate\n'), ((8618, 8662), 'langchain.evaluation.qa.QAEvalChain.from_llm', 'QAEvalChain.from_llm', ([], {'llm': 'llm', 'prompt': 'prompt'}), '(llm=llm, prompt=prompt)\n', (8638, 8662), False, 'from langchain.evaluation.qa import QAEvalChain\n'), ((9269, 9313), 'langchain.evaluation.qa.QAEvalChain.from_llm', 'QAEvalChain.from_llm', ([], {'llm': 'llm', 'prompt': 'prompt'}), '(llm=llm, prompt=prompt)\n', (9289, 9313), False, 'from langchain.evaluation.qa import QAEvalChain\n'), ((9654, 9708), 'Utilities.azureBlob.getBlob', 'getBlob', (['blobConnectionString', 'blobContainer', 'blobName'], {}), '(blobConnectionString, blobContainer, blobName)\n', (9661, 9708), False, 'from Utilities.azureBlob import getBlob, getFullPath\n'), ((9992, 10036), 'logging.info', 'logging.info', (["('File created ' + downloadPath)"], {}), "('File created ' + downloadPath)\n", (10004, 10036), False, 'import logging, json, os\n'), ((10164, 10222), 'Utilities.azureBlob.getFullPath', 'getFullPath', (['blobConnectionString', 'blobContainer', 'blobName'], {}), '(blobConnectionString, blobContainer, blobName)\n', (10175, 10222), False, 'from Utilities.azureBlob import getBlob, getFullPath\n'), ((10566, 10677), 'pandas.DataFrame', 'pd.DataFrame', ([], {'columns': "['question', 'answer', 'predictedAnswer', 'answerScore', 'retrievalScore',\n    'latency']"}), "(columns=['question', 'answer', 'predictedAnswer',\n    'answerScore', 'retrievalScore', 'latency'])\n", (10578, 10677), True, 'import pandas as pd\n'), ((13425, 13490), 'logging.info', 'logging.info', (['"""Python HTTP trigger function processed a request."""'], {}), "('Python HTTP trigger function processed a request.')\n", (13437, 13490), False, 'import logging, json, os\n'), ((14167, 14252), 'Utilities.evaluator.createEvaluatorResultIndex', 'createEvaluatorResultIndex', (['SearchService', 'SearchKey', 'evaluatorRunResultIndexName'], {}), '(SearchService, SearchKey,\n    evaluatorRunResultIndexName)\n', (14193, 14252), False, 'from Utilities.evaluator import createEvaluatorResultIndex, searchEvaluatorRunIdIndex\n'), ((14312, 14408), 'Utilities.evaluator.searchEvaluatorRunIdIndex', 'searchEvaluatorRunIdIndex', (['SearchService', 'SearchKey', 'evaluatorRunResultIndexName', 'documentId'], {}), '(SearchService, SearchKey,\n    evaluatorRunResultIndexName, documentId)\n', (14337, 14408), False, 'from Utilities.evaluator import createEvaluatorResultIndex, searchEvaluatorRunIdIndex\n'), ((9741, 9762), 'tempfile.gettempdir', 'tempfile.gettempdir', ([], {}), '()\n', (9760, 9762), False, 'import tempfile\n'), ((10090, 10118), 'langchain.document_loaders.PDFMinerLoader', 'PDFMinerLoader', (['downloadPath'], {}), '(downloadPath)\n', (10104, 10118), False, 'from langchain.document_loaders import PDFMinerLoader, UnstructuredFileLoader\n'), ((13563, 13741), 'langchain.chat_models.AzureChatOpenAI', 'AzureChatOpenAI', ([], {'azure_endpoint': 'OpenAiEndPoint', 'api_version': 'OpenAiVersion', 'azure_deployment': 'OpenAiChat', 'temperature': 'temperature', 'api_key': 'OpenAiKey', 'max_tokens': 'tokenLength'}), '(azure_endpoint=OpenAiEndPoint, api_version=OpenAiVersion,\n    azure_deployment=OpenAiChat, temperature=temperature, api_key=OpenAiKey,\n    max_tokens=tokenLength)\n', (13578, 13741), False, 'from langchain.chat_models import AzureChatOpenAI, ChatOpenAI\n'), ((13891, 13921), 'logging.info', 'logging.info', (['"""LLM Setup done"""'], {}), "('LLM Setup done')\n", (13903, 13921), False, 'import logging, json, os\n'), ((9806, 9827), 'tempfile.gettempdir', 'tempfile.gettempdir', ([], {}), '()\n', (9825, 9827), False, 'import tempfile\n'), ((9970, 9986), 'logging.error', 'logging.error', (['e'], {}), '(e)\n', (9983, 9986), False, 'import logging, json, os\n'), ((10897, 10908), 'time.time', 'time.time', ([], {}), '()\n', (10906, 10908), False, 'import time\n'), ((11079, 11090), 'time.time', 'time.time', ([], {}), '()\n', (11088, 11090), False, 'import time\n'), ((13981, 14095), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': 'temperature', 'api_key': 'OpenAiApiKey', 'model_name': '"""gpt-3.5-turbo"""', 'max_tokens': 'tokenLength'}), "(temperature=temperature, api_key=OpenAiApiKey, model_name=\n    'gpt-3.5-turbo', max_tokens=tokenLength)\n", (13991, 14095), False, 'from langchain.chat_models import AzureChatOpenAI, ChatOpenAI\n'), ((14452, 14464), 'uuid.uuid4', 'uuid.uuid4', ([], {}), '()\n', (14462, 14464), False, 'import uuid\n'), ((12809, 12831), 'pandas.DataFrame', 'pd.DataFrame', (['dfOutput'], {}), '(dfOutput)\n', (12821, 12831), True, 'import pandas as pd\n'), ((14735, 14894), 'Utilities.evaluator.searchEvaluatorRunIndex', 'searchEvaluatorRunIndex', (['SearchService', 'SearchKey', 'evaluatorRunResultIndexName', 'documentId', 'retrieverType', 'promptStyle', 'splitMethod', 'chunkSize', 'overlap'], {}), '(SearchService, SearchKey,\n    evaluatorRunResultIndexName, documentId, retrieverType, promptStyle,\n    splitMethod, chunkSize, overlap)\n', (14758, 14894), False, 'from Utilities.evaluator import searchEvaluatorRunIndex, createEvaluatorRunIndex, getEvaluatorResult\n'), ((15269, 15805), 'Utilities.cogSearchVsRetriever.CognitiveSearchVsRetriever', 'CognitiveSearchVsRetriever', ([], {'contentKey': '"""contentVector"""', 'serviceName': 'SearchService', 'apiKey': 'SearchKey', 'indexName': 'evaluatorDataIndexName', 'topK': 'topK', 'splitMethod': 'splitMethod', 'model': 'model', 'chunkSize': 'chunkSize', 'overlap': 'overlap', 'openAiEndPoint': 'OpenAiEndPoint', 'openAiKey': 'OpenAiKey', 'openAiVersion': 'OpenAiVersion', 'openAiApiKey': 'OpenAiApiKey', 'documentId': 'documentId', 'openAiEmbedding': 'OpenAiEmbedding', 'returnFields': "['id', 'content', 'sourceFile', 'splitMethod', 'chunkSize', 'overlap',\n    'model', 'modelType', 'documentId']"}), "(contentKey='contentVector', serviceName=\n    SearchService, apiKey=SearchKey, indexName=evaluatorDataIndexName, topK\n    =topK, splitMethod=splitMethod, model=model, chunkSize=chunkSize,\n    overlap=overlap, openAiEndPoint=OpenAiEndPoint, openAiKey=OpenAiKey,\n    openAiVersion=OpenAiVersion, openAiApiKey=OpenAiApiKey, documentId=\n    documentId, openAiEmbedding=OpenAiEmbedding, returnFields=['id',\n    'content', 'sourceFile', 'splitMethod', 'chunkSize', 'overlap', 'model',\n    'modelType', 'documentId'])\n", (15295, 15805), False, 'from Utilities.cogSearchVsRetriever import CognitiveSearchVsRetriever\n'), ((16345, 16472), 'langchain.chains.RetrievalQA.from_chain_type', 'RetrievalQA.from_chain_type', ([], {'llm': 'llm', 'chain_type': '"""stuff"""', 'retriever': 'retriever', 'chain_type_kwargs': "{'prompt': qaChainPrompt}"}), "(llm=llm, chain_type='stuff', retriever=\n    retriever, chain_type_kwargs={'prompt': qaChainPrompt})\n", (16372, 16472), False, 'from langchain.chains import RetrievalQA\n'), ((18132, 18219), 'Utilities.evaluator.indexDocs', 'indexDocs', (['SearchService', 'SearchKey', 'evaluatorRunResultIndexName', 'runEvaluationData'], {}), '(SearchService, SearchKey, evaluatorRunResultIndexName,\n    runEvaluationData)\n', (18141, 18219), False, 'from Utilities.evaluator import indexDocs\n'), ((15206, 15218), 'uuid.uuid4', 'uuid.uuid4', ([], {}), '()\n', (15216, 15218), False, 'import uuid\n'), ((17870, 17910), 'json.dumps', 'json.dumps', (["runEvaluation['answerScore']"], {}), "(runEvaluation['answerScore'])\n", (17880, 17910), False, 'import json\n'), ((17962, 18005), 'json.dumps', 'json.dumps', (["runEvaluation['retrievalScore']"], {}), "(runEvaluation['retrievalScore'])\n", (17972, 18005), False, 'import json\n'), ((17133, 17145), 'uuid.uuid4', 'uuid.uuid4', ([], {}), '()\n', (17143, 17145), False, 'import uuid\n')]