import logging
from dotenv import load_dotenv
from llama_index import VectorStoreIndex
import pandas as pd

from ragas.metrics import answer_relevancy
from ragas.llama_index import evaluate
from ragas.llms import LangchainLLM

from langchain.chat_models import AzureChatOpenAI
from langchain.embeddings import AzureOpenAIEmbeddings


from app.llama_index.vector_store import setup_vector_store
from app.llama_index.llm import setup_service_context

from app.utils.env import get_env_variable
from app.eval.constants import (
    DATASET_JSON_PATH,
    EVAL_METRICS,
    EVAL_VECTOR_STORE_NAME,
    SERVICE_CONTEXT_VERSION,
)
from app.eval.dataset_generation import generate_ragas_qr_pairs


def setup_ragas_llm():
    load_dotenv()
    try:
        api_key = get_env_variable("OPENAI_API_KEY")
        api_version = get_env_variable("OPENAI_API_VERSION")
        deployment_name = get_env_variable("OPENAI_DEPLOYMENT_NAME")
    except EnvironmentError as e:
        raise e

    azure_model = AzureChatOpenAI(
        deployment_name=deployment_name,
        model=api_version,
        openai_api_key=api_key,
        openai_api_type="azure",
    )
    logging.info("Azure OpenAI model for Ragas successfully set up.")
    return LangchainLLM(azure_model)


def setup_ragas_embeddings():
    load_dotenv()
    try:
        deployment = get_env_variable("OPENAI_DEPLOYMENT_EMBEDDINGS")
        api_base = get_env_variable("OPENAI_API_BASE")
        api_key = get_env_variable("OPENAI_API_KEY")
        api_version = get_env_variable("OPENAI_API_VERSION")
    except EnvironmentError as e:
        raise e

    azure_embeddings = AzureOpenAIEmbeddings(
        azure_deployment=deployment,
        model="text-embedding-ada-002",
        openai_api_type="azure",
        openai_api_base=api_base,
        openai_api_key=api_key,
        openai_api_version=api_version,
    )
    logging.info("Azure OpenAI Embeddings for Ragas successfully set up.")
    return azure_embeddings


def run_ragas_evaluation():
    eval_questions, eval_answers = generate_ragas_qr_pairs(DATASET_JSON_PATH)
    eval_embeddings = setup_ragas_embeddings()
    eval_llm = setup_ragas_llm()
    eval_vector_store = setup_vector_store(EVAL_VECTOR_STORE_NAME)
    eval_service_context = setup_service_context(SERVICE_CONTEXT_VERSION, azure=True)
    index = VectorStoreIndex.from_vector_store(
        vector_store=eval_vector_store, service_context=eval_service_context
    )
    query_engine = index.as_query_engine()
    logging.info("Ragas evaluation successfully set up.")

    metrics = EVAL_METRICS
    answer_relevancy.embeddings = eval_embeddings
    for m in metrics:
        m.__setattr__("llm", eval_llm)
        m.__setattr__("embeddings", eval_embeddings)
    logging.info("Ragas metrics successfully set up.")
    result = evaluate(query_engine, metrics, eval_questions, eval_answers)
    logging.info("Ragas evaluation successfully finished.")
    df = result.to_pandas()
    df.to_csv("app/eval/eval_data/ragas_eval.csv", index=False)
    logging.info("Ragas evaluation successfully saved to csv file.")
    eval = pd.read_csv("app/eval/eval_data/ragas_eval.csv", sep=",")
    logging.info("Ragas evaluation successfully finished.")
    return eval

langchain.embeddings.AzureOpenAIEmbeddings langchain.chat_models.AzureChatOpenAI
[((718, 731), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (729, 731), False, 'from dotenv import load_dotenv\n'), ((993, 1113), 'langchain.chat_models.AzureChatOpenAI', 'AzureChatOpenAI', ([], {'deployment_name': 'deployment_name', 'model': 'api_version', 'openai_api_key': 'api_key', 'openai_api_type': '"""azure"""'}), "(deployment_name=deployment_name, model=api_version,\n    openai_api_key=api_key, openai_api_type='azure')\n", (1008, 1113), False, 'from langchain.chat_models import AzureChatOpenAI\n'), ((1153, 1218), 'logging.info', 'logging.info', (['"""Azure OpenAI model for Ragas successfully set up."""'], {}), "('Azure OpenAI model for Ragas successfully set up.')\n", (1165, 1218), False, 'import logging\n'), ((1230, 1255), 'ragas.llms.LangchainLLM', 'LangchainLLM', (['azure_model'], {}), '(azure_model)\n', (1242, 1255), False, 'from ragas.llms import LangchainLLM\n'), ((1292, 1305), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (1303, 1305), False, 'from dotenv import load_dotenv\n'), ((1628, 1827), 'langchain.embeddings.AzureOpenAIEmbeddings', 'AzureOpenAIEmbeddings', ([], {'azure_deployment': 'deployment', 'model': '"""text-embedding-ada-002"""', 'openai_api_type': '"""azure"""', 'openai_api_base': 'api_base', 'openai_api_key': 'api_key', 'openai_api_version': 'api_version'}), "(azure_deployment=deployment, model=\n    'text-embedding-ada-002', openai_api_type='azure', openai_api_base=\n    api_base, openai_api_key=api_key, openai_api_version=api_version)\n", (1649, 1827), False, 'from langchain.embeddings import AzureOpenAIEmbeddings\n'), ((1877, 1947), 'logging.info', 'logging.info', (['"""Azure OpenAI Embeddings for Ragas successfully set up."""'], {}), "('Azure OpenAI Embeddings for Ragas successfully set up.')\n", (1889, 1947), False, 'import logging\n'), ((2041, 2083), 'app.eval.dataset_generation.generate_ragas_qr_pairs', 'generate_ragas_qr_pairs', (['DATASET_JSON_PATH'], {}), '(DATASET_JSON_PATH)\n', (2064, 2083), False, 'from app.eval.dataset_generation import generate_ragas_qr_pairs\n'), ((2188, 2230), 'app.llama_index.vector_store.setup_vector_store', 'setup_vector_store', (['EVAL_VECTOR_STORE_NAME'], {}), '(EVAL_VECTOR_STORE_NAME)\n', (2206, 2230), False, 'from app.llama_index.vector_store import setup_vector_store\n'), ((2258, 2316), 'app.llama_index.llm.setup_service_context', 'setup_service_context', (['SERVICE_CONTEXT_VERSION'], {'azure': '(True)'}), '(SERVICE_CONTEXT_VERSION, azure=True)\n', (2279, 2316), False, 'from app.llama_index.llm import setup_service_context\n'), ((2329, 2437), 'llama_index.VectorStoreIndex.from_vector_store', 'VectorStoreIndex.from_vector_store', ([], {'vector_store': 'eval_vector_store', 'service_context': 'eval_service_context'}), '(vector_store=eval_vector_store,\n    service_context=eval_service_context)\n', (2363, 2437), False, 'from llama_index import VectorStoreIndex\n'), ((2495, 2548), 'logging.info', 'logging.info', (['"""Ragas evaluation successfully set up."""'], {}), "('Ragas evaluation successfully set up.')\n", (2507, 2548), False, 'import logging\n'), ((2745, 2795), 'logging.info', 'logging.info', (['"""Ragas metrics successfully set up."""'], {}), "('Ragas metrics successfully set up.')\n", (2757, 2795), False, 'import logging\n'), ((2809, 2870), 'ragas.llama_index.evaluate', 'evaluate', (['query_engine', 'metrics', 'eval_questions', 'eval_answers'], {}), '(query_engine, metrics, eval_questions, eval_answers)\n', (2817, 2870), False, 'from ragas.llama_index import evaluate\n'), ((2875, 2930), 'logging.info', 'logging.info', (['"""Ragas evaluation successfully finished."""'], {}), "('Ragas evaluation successfully finished.')\n", (2887, 2930), False, 'import logging\n'), ((3027, 3091), 'logging.info', 'logging.info', (['"""Ragas evaluation successfully saved to csv file."""'], {}), "('Ragas evaluation successfully saved to csv file.')\n", (3039, 3091), False, 'import logging\n'), ((3103, 3160), 'pandas.read_csv', 'pd.read_csv', (['"""app/eval/eval_data/ragas_eval.csv"""'], {'sep': '""","""'}), "('app/eval/eval_data/ragas_eval.csv', sep=',')\n", (3114, 3160), True, 'import pandas as pd\n'), ((3165, 3220), 'logging.info', 'logging.info', (['"""Ragas evaluation successfully finished."""'], {}), "('Ragas evaluation successfully finished.')\n", (3177, 3220), False, 'import logging\n'), ((759, 793), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (775, 793), False, 'from app.utils.env import get_env_variable\n'), ((816, 854), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_API_VERSION"""'], {}), "('OPENAI_API_VERSION')\n", (832, 854), False, 'from app.utils.env import get_env_variable\n'), ((881, 923), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_DEPLOYMENT_NAME"""'], {}), "('OPENAI_DEPLOYMENT_NAME')\n", (897, 923), False, 'from app.utils.env import get_env_variable\n'), ((1336, 1384), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_DEPLOYMENT_EMBEDDINGS"""'], {}), "('OPENAI_DEPLOYMENT_EMBEDDINGS')\n", (1352, 1384), False, 'from app.utils.env import get_env_variable\n'), ((1404, 1439), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_API_BASE"""'], {}), "('OPENAI_API_BASE')\n", (1420, 1439), False, 'from app.utils.env import get_env_variable\n'), ((1458, 1492), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (1474, 1492), False, 'from app.utils.env import get_env_variable\n'), ((1515, 1553), 'app.utils.env.get_env_variable', 'get_env_variable', (['"""OPENAI_API_VERSION"""'], {}), "('OPENAI_API_VERSION')\n", (1531, 1553), False, 'from app.utils.env import get_env_variable\n')]