app_old.py

import os
import shutil
import streamlit as st
from fpdf import FPDF
from chromadb import Client
from chromadb.config import Settings
import json
import chromadb
from langchain_community.utilities import SerpAPIWrapper
from llama_index.core import VectorStoreIndex
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
from langchain_groq import ChatGroq
from langchain.chains import LLMChain
from langchain.agents import AgentType, Tool, initialize_agent, AgentExecutor
from llama_parse import LlamaParse
from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_huggingface import HuggingFaceEmbeddings
from llama_index.core import SimpleDirectoryReader
from dotenv import load_dotenv, find_dotenv
from streamlit_chat import message
from langchain_community.vectorstores import Chroma
from langchain_community.utilities import SerpAPIWrapper
from langchain.chains import RetrievalQA
from langchain_community.document_loaders import DirectoryLoader
from langchain_community.document_loaders import PyMuPDFLoader
from langchain_community.document_loaders import UnstructuredXMLLoader
from langchain_community.document_loaders import CSVLoader
from langchain.prompts import PromptTemplate
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.memory import ConversationBufferMemory
from langchain.prompts import PromptTemplate
import joblib
import nltk
from dotenv import load_dotenv, find_dotenv
import uuid
from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain_community.vectorstores import Chroma
from langchain.chat_models import ChatOpenAI
from langchain.embeddings import OpenAIEmbeddings
from langchain.prompts import PromptTemplate
from yachalk import chalk
from langchain.vectorstores import PGVector
from langchain.document_loaders import PyPDFLoader, UnstructuredPDFLoader, PyPDFium2Loader
from langchain.document_loaders import PyPDFDirectoryLoader
## Import all the chains.
from chains_v2.create_questions import QuestionCreationChain
from chains_v2.most_pertinent_question import MostPertinentQuestion
from chains_v2.retrieval_qa import retrieval_qa
from chains_v2.research_compiler import research_compiler
from chains_v2.question_atomizer import QuestionAtomizer
from chains_v2.refine_answer import RefineAnswer
## Import all the helpers.
from helpers.response_helpers import result2QuestionsList
from helpers.response_helpers import qStr2Dict
from helpers.questions_helper import getAnsweredQuestions
from helpers.questions_helper import getUnansweredQuestions
from helpers.questions_helper import getSubQuestions
from helpers.questions_helper import getHopQuestions
from helpers.questions_helper import getLastQuestionId
from helpers.questions_helper import markAnswered
from helpers.questions_helper import getQuestionById

import nest_asyncio  # noqa: E402
nest_asyncio.apply()

load_dotenv()
load_dotenv(find_dotenv())

nltk.download('averaged_perceptron_tagger_eng') 

os.environ["TOKENIZERS_PARALLELISM"] = "false"
SERPAPI_API_KEY = os.environ["SERPAPI_API_KEY"]
GOOGLE_CSE_ID = os.environ["GOOGLE_CSE_ID"]
GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"]
LLAMA_PARSE_API_KEY = os.environ["LLAMA_PARSE_API_KEY"]
HUGGINGFACEHUB_API_TOKEN = os.environ["HUGGINGFACEHUB_API_TOKEN"]
LANGCHAIN_API_KEY = os.environ["LANGCHAIN_API_KEY"]
LANGCHAIN_ENDPOINT = os.environ["LANGCHAIN_ENDPOINT"]
LANGCHAIN_PROJECT = os.environ["LANGCHAIN_PROJECT"]
groq_api_key=os.getenv('GROQ_API_KEY')

st.set_page_config(layout="wide")

css = """
<style>
    [data-testid="stAppViewContainer"] {
        background-color: #f8f9fa; /* Very light grey */
    }
    [data-testid="stSidebar"] {
        background-color: white;
        color: black;
    }
    [data-testid="stAppViewContainer"] * {
        color: black; /* Ensure all text is black */
    }
    button {
        background-color: #add8e6; /* Light blue for primary buttons */
        color: black;
        border: 2px solid green; /* Green border */
    }
    button:hover {
        background-color: #87ceeb; /* Slightly darker blue on hover */
    }

    button:active {
        outline: 2px solid green; /* Green outline when the button is pressed */
        outline-offset: 2px; /* Space between button and outline */
    }

    .stButton>button:first-child {
        background-color: #add8e6; /* Light blue for primary buttons */
        color: black;
    }
    .stButton>button:first-child:hover {
        background-color: #87ceeb; /* Slightly darker blue on hover */
    }
    .stButton>button:nth-child(2) {
        background-color: #b0e0e6; /* Even lighter blue for secondary buttons */
        color: black;
    }
    .stButton>button:nth-child(2):hover {
        background-color: #add8e6; /* Slightly darker blue on hover */
    }
    [data-testid="stFileUploadDropzone"] {
        background-color: white; /* White background for file upload */
    }
    [data-testid="stFileUploadDropzone"] .stDropzone, [data-testid="stFileUploadDropzone"] .stDropzone input {
        color: black; /* Ensure file upload text is black */
    }
    
    .stButton>button:active {
        outline: 2px solid green; /* Green outline when the button is pressed */
        outline-offset: 2px;
    }
</style>
"""
def load_credentials(filepath):
    with open(filepath, 'r') as file:
        return json.load(file)

# Load credentials from 'credentials.json'
credentials = load_credentials('Assets/credentials.json')

# Initialize session state if not already done
if 'logged_in' not in st.session_state:
    st.session_state.logged_in = False
    st.session_state.username = ''

# Function to handle login
def login(username, password):
    if username in credentials and credentials[username] == password:
        st.session_state.logged_in = True
        st.session_state.username = username
        st.rerun()  # Rerun to reflect login state
    else:
        st.session_state.logged_in = False
        st.session_state.username = ''
        st.error("Invalid username or password.")

# Function to handle logout
def logout():
    st.session_state.logged_in = False
    st.session_state.username = ''
    st.rerun()  # Rerun to reflect logout state

#--------------
## Define log printers

def print_iteration(current_iteration):
    print(
        chalk.bg_yellow_bright.black.bold(
            f"\n   Iteration - {current_iteration}  ▷▶  \n"
        )
    )


def print_unanswered_questions(unanswered):
    print(
        chalk.cyan_bright("** Unanswered Questions **"),
        chalk.cyan("".join([f"\n'{q['id']}. {q['question']}'" for q in unanswered])),
    )


def print_next_question(current_question_id, current_question):
    print(
        chalk.magenta.bold("** 🤔 Next Questions I must ask: **\n"),
        chalk.magenta(current_question_id),
        chalk.magenta(current_question["question"]),
    )


def print_answer(current_question):
    print(
        chalk.yellow_bright.bold("** Answer **\n"),
        chalk.yellow_bright(current_question["answer"]),
    )


def print_final_answer(answerpad):
    print(
        chalk.white("** Refined Answer **\n"),
        chalk.white(answerpad[-1]),
    )


def print_max_iterations():
    print(
        chalk.bg_yellow_bright.black.bold(
            "\n ✔✔  Max Iterations Reached. Compiling the results ...\n"
        )
    )


def print_result(result):
    print(chalk.italic.white_bright((result["text"])))


def print_sub_question(q):
    print(chalk.magenta.bold(f"** Sub Question **\n{q['question']}\n{q['answer']}\n"))
#--------------
## ---- The researcher ----- ##

class Agent:
    ## Create chains
    def __init__(self, agent_settings, scratchpad, store, verbose):
        self.store = store
        self.scratchpad = scratchpad
        self.agent_settings = agent_settings
        self.verbose = verbose
        self.question_creation_chain = QuestionCreationChain.from_llm(
            language_model(
                temperature=self.agent_settings["question_creation_temperature"]
            ),
            verbose=self.verbose,
        )
        self.question_atomizer = QuestionAtomizer.from_llm(
            llm=language_model(
                temperature=self.agent_settings["question_atomizer_temperature"]
            ),
            verbose=self.verbose,
        )
        self.most_pertinent_question = MostPertinentQuestion.from_llm(
            language_model(
                temperature=self.agent_settings["question_creation_temperature"]
            ),
            verbose=self.verbose,
        )
        self.refine_answer = RefineAnswer.from_llm(
            language_model(
                temperature=self.agent_settings["refine_answer_temperature"]
            ),
            verbose=self.verbose,
        )

    def run(self, question):
        ## Step 0. Prepare the initial set of questions
        atomized_questions_response = self.question_atomizer.run(
            question=question,
            num_questions=self.agent_settings["num_atomistic_questions"],
        )

        self.scratchpad["questions"] += result2QuestionsList(
            question_response=atomized_questions_response,
            type="subquestion",
            status="unanswered",
        )

        for q in self.scratchpad["questions"]:
            q["answer"], q["documents"] = retrieval_qa(
                llm=language_model(
                    temperature=self.agent_settings["qa_temperature"],
                    verbose=self.verbose,
                ),
                retriever=self.store.as_retriever(
                    search_type="mmr", search_kwargs={"k": 5, "fetch_k": 10}
                ),
                question=q["question"],
                answer_length=self.agent_settings["intermediate_answers_length"],
                verbose=self.verbose,
            )
            q["status"] = "answered"
            print_sub_question(q)

        
        current_context = "".join(
            f"\n{q['id']}. {q['question']}\n{q['answer']}\n"
            for q in self.scratchpad["questions"]
        )
        
        self.scratchpad["answerpad"] += [current_context]

        current_iteration = 0

        while True:
            current_iteration += 1
            print_iteration(current_iteration)

            # STEP 1: create questions
            start_id = getLastQuestionId(self.scratchpad["questions"]) + 1
            questions_response = self.question_creation_chain.run(
                question=question,
                context=current_context,
                previous_questions=[
                    "".join(f"\n{q['question']}") for q in self.scratchpad["questions"]
                ],
                num_questions=self.agent_settings["num_questions_per_iteration"],
                start_id=start_id,
            )
            self.scratchpad["questions"] += result2QuestionsList(
                question_response=questions_response,
                type="hop",
                status="unanswered",
            )

            # STEP 2: Choose question for current iteration
            unanswered = getUnansweredQuestions(self.scratchpad["questions"])
            unanswered_questions_prompt = self.unanswered_questions_prompt(unanswered)
            print_unanswered_questions(unanswered)
            response = self.most_pertinent_question.run(
                original_question=question,
                unanswered_questions=unanswered_questions_prompt,
            )
            current_question_dict = qStr2Dict(question=response)
            current_question_id = current_question_dict["id"]
            current_question = getQuestionById(
                self.scratchpad["questions"], current_question_id
            )
            print_next_question(current_question_id, current_question)

            # STEP 3: Answer the question
            current_question["answer"], current_question["documents"] = retrieval_qa(
                llm=language_model(
                    temperature=self.agent_settings["qa_temperature"],
                    verbose=self.verbose,
                ),
                retriever=self.store.as_retriever(
                    search_type="mmr", search_kwargs={"k": 5, "fetch_k": 10}
                ),
                question=current_question["question"],
                answer_length=self.agent_settings["intermediate_answers_length"],
                verbose=self.verbose,
            )
            markAnswered(self.scratchpad["questions"], current_question_id)
            print_answer(current_question)
            current_context = current_question["answer"]

            ## STEP 4: refine the answer
            refinement_context = current_question["question"] + "\n" + current_context
            refine_answer = self.refine_answer.run(
                question=question,
                context=refinement_context,
                answer=self.get_latest_answer(),
            )
            self.scratchpad["answerpad"] += [refine_answer]
            print_final_answer(self.scratchpad["answerpad"])

            if current_iteration > self.agent_settings["max_iterations"]:
                print_max_iterations()
                break

    def unanswered_questions_prompt(self, unanswered):
        return (
            "[" + "".join([f"\n{q['id']}. {q['question']}" for q in unanswered]) + "]"
        )

    def notes_prompt(self, answered_questions):
        return "".join(
            [
                f"{{ Question: {q['question']}, Answer: {q['answer']} }}"
                for q in answered_questions
            ]
        )

    def get_latest_answer(self):
        answers = self.scratchpad["answerpad"]
        answer = answers[-1] if answers else ""
        return answer
    
#--------------
# If not logged in, show login form
if not st.session_state.logged_in:
    st.sidebar.write("Login")
    username = st.sidebar.text_input('Username')
    password = st.sidebar.text_input('Password', type='password')
    if st.sidebar.button('Login'):
        login(username, password)
    # Stop the script here if the user is not logged in
    st.stop()


# If logged in, show logout button and main content
st.sidebar.image('StratXcel.png', width=150)
if st.session_state.logged_in:
    st.sidebar.write(f"Welcome, {st.session_state.username}!")
    if st.sidebar.button('Logout'):
        logout()

st.write(css, unsafe_allow_html=True)

company_document = st.sidebar.toggle("Company document", False)
financial_document = st.sidebar.toggle("Financial document", False)
intercreditor_document = st.sidebar.toggle("Intercreditor document", False)

#------------- 
llm=ChatGroq(groq_api_key=groq_api_key,
             model_name="Llama-3.1-70b-Versatile", temperature = 0.0, streaming=True)   

Llama = "Llama-3.1-70b-Versatile"
def language_model(
    model_name: str = Llama, temperature: float = 0, verbose: bool = False
    ):
    llm=ChatGroq(groq_api_key=groq_api_key, model_name=model_name, temperature=temperature, verbose=verbose)    
    return llm
#--------------
doc_retriever_company = None
doc_retriever_financials = None
doc_retriever_intercreditor = None

#--------------

#@st.cache_data
def load_or_parse_data_company():
    data_file = "./data/parsed_data_company.pkl"

    parsingInstructionUber10k = """The provided documents are company law documents of a company.
    They contain detailed information about the rights and obligations of the company and its shareholders and contracting parties.
    They also contain procedures for dispute resolution, voting, control priority and exit and sale situations. 
    You must never provide false legal or financial information. Use only the information included in the context documents.
    Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""

    parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
                        result_type="markdown",
                        parsing_instruction=parsingInstructionUber10k,
                        max_timeout=5000,
                        gpt4o_mode=True,
                        )

    file_extractor = {".pdf": parser}
    reader = SimpleDirectoryReader("./Corporate_Documents", file_extractor=file_extractor)
    documents = reader.load_data()

    print("Saving the parse results in .pkl format ..........")
    joblib.dump(documents, data_file)

    # Set the parsed data to the variable
    parsed_data_company = documents

    return parsed_data_company

#@st.cache_data
def load_or_parse_data_financial():
    data_file = "./data/parsed_data_financial.pkl"

    parsingInstructionUber10k = """The provided documents are financial law documents of a company.
    They contain detailed information about the rights and obligations of the company and its creditors.
    They also contain procedures for acceleration of debt, sale of security, enforcement, use of creditor control, priority and distribution of assets. 
    You must never provide false legal or financial information. Use only the information included in the context documents.
    Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""

    parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
                        result_type="markdown",
                        parsing_instruction=parsingInstructionUber10k,
                        max_timeout=5000,
                        gpt4o_mode=True,
                        )

    file_extractor = {".pdf": parser}
    reader = SimpleDirectoryReader("./Financial_Documents", file_extractor=file_extractor)
    documents = reader.load_data()

    print("Saving the parse results in .pkl format ..........")
    joblib.dump(documents, data_file)

    # Set the parsed data to the variable
    parsed_data_financial = documents

    return parsed_data_financial

#--------------

#@st.cache_data
def load_or_parse_data_intercreditor():
    data_file = "./data/parsed_data_intercreditor.pkl"

    parsingInstructionUber10k = """The provided documents are intercreditor agreements a company .
    They contain detailed information about the rights and obligations of the company and its creditors and creditor groups.
    They also contain procedures for acceleration of debt, sale of security, enforcement, use of creditor control, priority and distribution of assets. 
    You must never provide false legal or financial information. Use only the information included in the context documents.
    Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""

    parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
                        result_type="markdown",
                        parsing_instruction=parsingInstructionUber10k,
                        max_timeout=5000,
                        gpt4o_mode=True,
                        )

    file_extractor = {".pdf": parser}
    reader = SimpleDirectoryReader("./Intercreditor_Documents", file_extractor=file_extractor)
    documents = reader.load_data()

    print("Saving the parse results in .pkl format ..........")
    joblib.dump(documents, data_file)

    # Set the parsed data to the variable
    parsed_data_financial = documents

    return parsed_data_financial
#--------------
# Create vector database

@st.cache_resource
def create_vector_database_company():
    # Call the function to either load or parse the data
    llama_parse_documents = load_or_parse_data_company()

    with open('data/output_company.md', 'a') as f:  # Open the file in append mode ('a')
        for doc in llama_parse_documents:
            f.write(doc.text + '\n')

    markdown_path = "data/output_company.md"
    loader = UnstructuredMarkdownLoader(markdown_path)
    documents = loader.load()
    # Split loaded documents into chunks
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=30)
    docs = text_splitter.split_documents(documents)

    #len(docs)
    print(f"length of documents loaded: {len(documents)}")
    print(f"total number of document chunks generated :{len(docs)}")
    embed_model = HuggingFaceEmbeddings()
    print('Vector DB not yet created !')

    vs = Chroma.from_documents(
        documents=docs,
        embedding=embed_model,
        collection_name="rag",
    )

    doc_retriever_company = vs
    #doc_retriever_company = vs.as_retriever()
    
    print('Vector DB created successfully !')
    return doc_retriever_company

@st.cache_resource
def create_vector_database_financial():
    # Call the function to either load or parse the data
    llama_parse_documents = load_or_parse_data_financial()

    with open('data/output_financials.md', 'a') as f:  # Open the file in append mode ('a')
        for doc in llama_parse_documents:
            f.write(doc.text + '\n')

    markdown_path = "data/output_financials.md"
    loader = UnstructuredMarkdownLoader(markdown_path)
    documents = loader.load()
    # Split loaded documents into chunks
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15)
    docs = text_splitter.split_documents(documents)

    print(f"length of documents loaded: {len(documents)}")
    print(f"total number of document chunks generated :{len(docs)}")
    embed_model = HuggingFaceEmbeddings()

    vs = Chroma.from_documents(
        documents=docs,
        embedding=embed_model,
        collection_name="rag"
    )
    doc_retriever_financial = vs
    #doc_retriever_financial = vs.as_retriever()

    print('Vector DB created successfully !')
    return doc_retriever_financial

#--------------

@st.cache_resource
def create_vector_database_intercreditor():
    # Call the function to either load or parse the data
    llama_parse_documents = load_or_parse_data_intercreditor()

    with open('data/output_intercreditor.md', 'a') as f:  # Open the file in append mode ('a')
        for doc in llama_parse_documents:
            f.write(doc.text + '\n')

    markdown_path = "data/output_intercreditor.md"
    loader = UnstructuredMarkdownLoader(markdown_path)
    documents = loader.load()
    # Split loaded documents into chunks
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15)
    docs = text_splitter.split_documents(documents)

    print(f"length of documents loaded: {len(documents)}")
    print(f"total number of document chunks generated :{len(docs)}")
    embed_model = HuggingFaceEmbeddings()

    vs = Chroma.from_documents(
        documents=docs,
        embedding=embed_model,
        collection_name="rag"
    )
    doc_retriever_intercreditor = vs
    #doc_retriever_intercreditor = vs.as_retriever()

    print('Vector DB created successfully !')
    return doc_retriever_intercreditor

#--------------
legal_analysis_button_key = "legal_strategy_button"

#---------------
def delete_files_and_folders(folder_path):
    for root, dirs, files in os.walk(folder_path, topdown=False):
        for file in files:
            try:
                os.unlink(os.path.join(root, file))
            except Exception as e:
                st.error(f"Error deleting {os.path.join(root, file)}: {e}")
        for dir in dirs:
            try:
                os.rmdir(os.path.join(root, dir))
            except Exception as e:
                st.error(f"Error deleting directory {os.path.join(root, dir)}: {e}")
#---------------

if company_document:
    uploaded_files_ESG = st.sidebar.file_uploader("Choose company law documents", accept_multiple_files=True, key="company_files")
    for uploaded_file in uploaded_files_ESG:
        st.write("filename:", uploaded_file.name)
        def save_uploadedfile(uploadedfile):
            with open(os.path.join("Corporate_Documents",uploadedfile.name),"wb") as f:
                f.write(uploadedfile.getbuffer())
            return st.success("Saved File:{} to Company_Documents".format(uploadedfile.name))
        save_uploadedfile(uploaded_file)

if financial_document:
    uploaded_files_financials = st.sidebar.file_uploader("Choose financial law documents", accept_multiple_files=True, key="financial_files")
    for uploaded_file in uploaded_files_financials:
        st.write("filename:", uploaded_file.name)
        def save_uploadedfile(uploadedfile):
            with open(os.path.join("Financial_Documents",uploadedfile.name),"wb") as f:
                f.write(uploadedfile.getbuffer())
            return st.success("Saved File:{} to Financial_Documents".format(uploadedfile.name))
        save_uploadedfile(uploaded_file)

if intercreditor_document:
    uploaded_files_intercreditor = st.sidebar.file_uploader("Choose intercreditor documents", accept_multiple_files=True, key="intercreditor_files")
    for uploaded_file in uploaded_files_intercreditor:
        st.write("filename:", uploaded_file.name)
        def save_uploadedfile(uploadedfile):
            with open(os.path.join("Intercreditor_Documents",uploadedfile.name),"wb") as f:
                f.write(uploadedfile.getbuffer())
            return st.success("Saved File:{} to Intercreditor_Documents".format(uploadedfile.name))
        save_uploadedfile(uploaded_file)
#---------------
def company_strategy():
    doc_retriever_company = create_vector_database_company().as_retriever()
    prompt_template = """<|system|>
    You are a seasoned attorney specilizing in company and corporate law and legal analysis. You write expert analyses for institutional investors. 
    Output must have sub-headings in bold font and be fluent.<|end|>
    <|user|>
    Answer the {question} based on the information you find in context: {context} <|end|>
    <|assistant|>""" 

    prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])

    qa = (
    {
        "context": doc_retriever_company,
        "question": RunnablePassthrough(),
    }
    | prompt
    | llm
    | StrOutputParser()
)   

    Corporate_answer_1 = qa.invoke("What provisions govern the appointment and removal of directors in the company? Outline the procedures and any required shareholder involvement in these processes.")

    Corporate_answer_2 = qa.invoke("Explain the company's share capital structure, including any provisions for different classes of shares and the rights attached to them. How are voting rights distributed among shareholders?")

    Corporate_answer_3 = qa.invoke("What restrictions or conditions are placed on the transfer or sale of shares in the company's articles of association or shareholders' agreements? Include any pre-emption rights or lock-in provisions.")

    Corporate_answer_4 = qa.invoke("Describe the rights and obligations of majority and minority shareholders as outlined in the company's shareholders' agreements. What protections are in place for minority shareholders?")

    Corporate_answer_5 = qa.invoke("What are the provisions for issuing new shares or increasing the company's capital? Detail any existing shareholder approval requirements or pre-emptive rights outlined in the company's governing documents.")

    Corporate_answer_6 = qa.invoke("Outline the procedures for decision-making in shareholder meetings, including quorum requirements and voting thresholds for ordinary and special resolutions. How are dissenting shareholders addressed in key decisions?")

    Corporate_answer_7 = qa.invoke("What mechanisms are in place for resolving shareholder disputes? Provide details on any arbitration or mediation clauses found in the company's articles or shareholders' agreements.")

    Corporate_answer_8 = qa.invoke("Describe the exit mechanisms available for shareholders, such as drag-along and tag-along rights, and the circumstances under which they can be triggered.")

    Corporate_answer_9 = qa.invoke("What rights do shareholders have to appoint or remove members of the board? Outline any requirements for shareholder approval in relation to board appointments or dismissals.")

    Corporate_answer_10 = qa.invoke("Explain any restrictions on the powers of the board as set out in the company's governing documents. Are there specific decisions that require shareholder approval or consultation?")

    Corporate_answer_11 = qa.invoke("What provisions are in place regarding dividends and the distribution of profits? How are dividend rights structured among different classes of shares, if applicable?")


    corporate_output = f"""**__Director Appointment and Removal:__** {Corporate_answer_1} \n\n 
**__Share Capital Structure and Voting Rights:__** {Corporate_answer_2} \n\n 
**__Restrictions on Share Transfer and Sale:__** {Corporate_answer_3} \n\n 
**__Rights and Obligations of Shareholders:__** {Corporate_answer_4} \n\n 
**__Issuing New Shares and Capital Increases:__** {Corporate_answer_5} \n\n 
**__Decision-Making in Shareholder Meetings:__** {Corporate_answer_6} \n\n 
**__Shareholder Dispute Resolution Mechanisms:__** {Corporate_answer_7} \n\n 
**__Exit Mechanisms for Shareholders:__** {Corporate_answer_8} \n\n 
**__Rights to Appoint or Remove Board Members:__** {Corporate_answer_9} \n\n 
**__Restrictions on the Board's Powers:__** {Corporate_answer_10} \n\n 
**__Dividend and Profit Distribution Provisions:__** {Corporate_answer_11}"""
    
    financial_output = corporate_output
    
    with open("company_analysis.txt", 'w') as file:
        file.write(financial_output)
    
    return financial_output

def financial_strategy():
    doc_retriever_financial = create_vector_database_financial().as_retriever()
    prompt_template = """<|system|>
    You are a seasoned attorney specializing in financial law and legal analysis. You write expert analyses for institutional investors. 
    Output must have fluent sub-headings in bold font.<|end|>
    <|user|>
    Answer the {question} based on the information you find in context: {context} <|end|>
    <|assistant|>""" 

    prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])

    qa = (
    {
        "context": doc_retriever_financial,
        "question": RunnablePassthrough(),
    }
    | prompt
    | llm
    | StrOutputParser()
)   

    Financial_answer_1 = qa.invoke("What are the parties of the agreements and key obligations of the borrower under the company's loan agreements? Describe any covenants or financial ratios the borrower must comply with.")

    Financial_answer_3 = qa.invoke("What provisions govern the occurrence of events of default under the company's loan and bond agreements? Include any cross-default or material adverse change clauses.")

    Financial_answer_4 = qa.invoke("Describe the rights of secured creditors under the company's security documents. What types of assets are secured, and how can creditors enforce their security in case of default?")

    Financial_answer_5 = qa.invoke("What are the acceleration clauses in the company's financial agreements? Under what conditions can creditors demand early repayment or terminate financing arrangements?")

    Financial_answer_6 = qa.invoke("Outline the procedures and requirements for enforcing security interests under the company's security documents. How do the rights of secured and unsecured creditors differ in this context?")

    Financial_answer_7 = qa.invoke("How are decisions related to enforcement or restructuring prioritized among creditors?")

    Financial_answer_8 = qa.invoke("Explain the company's obligations under any guarantees or indemnities provided to creditors. What are the limitations, if any, on the enforcement of these guarantees?")

    Financial_answer_9 = qa.invoke("Describe the rights of bondholders or lenders in the company's bond issuance agreements or loans. What are the procedures for creditor meetings, and how can creditors exercise their rights in the event of default?")

    Financial_answer_10 = qa.invoke("What protections are in place for junior creditors or subordinated debt holders in the company's financial agreements? How are their rights affected in the event of enforcement or restructuring?")

    Financial_answer_11 = qa.invoke("What are the company's obligations to provide financial information to creditors under its loan or bond agreements? How frequently must the company report, and what information is typically required?")


    financial_output = f"""**__Borrower Obligations and Covenants:__** {Financial_answer_1} \n\n 
**__Events of Default and Cross-Default Provisions:__** {Financial_answer_3} \n\n 
**__Rights of Secured Creditors and Enforcement of Security:__** {Financial_answer_4} \n\n 
**__Acceleration Clauses and Early Repayment Triggers:__** {Financial_answer_5} \n\n 
**__Enforcement of Security Interests:__** {Financial_answer_6} \n\n 
**__Intercreditor Decision-Making and Control:__** {Financial_answer_7} \n\n 
**__Guarantees and Indemnities Obligations:__** {Financial_answer_8} \n\n 
**__Rights of Bondholders and Default Procedures:__** {Financial_answer_9} \n\n 
**__Protections for Junior Creditors:__** {Financial_answer_10} \n\n 
**__Financial Reporting Obligations to Creditors:__** {Financial_answer_11}"""
        
    with open("financial_analysis.txt", 'w') as file:
        file.write(financial_output)

    return financial_output

def intercreditor_strategy():
    doc_retriever_intercreditor = create_vector_database_intercreditor().as_retriever()
    prompt_template = """<|system|>
    You are a seasoned attorney specilizing in financial law and legal analysis. You write expert analyses for institutional investors. 
    Output must have sub-headings in bold font and be fluent.<|end|>
    <|user|>
    Answer the {question} based on the information you find in context: {context} <|end|>
    <|assistant|>""" 

    prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])

    qa = (
    {
        "context": doc_retriever_intercreditor,
        "question": RunnablePassthrough(),
    }
    | prompt
    | llm
    | StrOutputParser()
)   

    Intercreditor_answer_1 = qa.invoke("What are the parties involved in the intercreditor agreements, and what are their key roles and obligations?")

    Intercreditor_answer_2 = qa.invoke("What provisions govern the ranking and priority of claims among creditors under the intercreditor agreements? Describe any subordination terms or waterfall clauses.")

    Intercreditor_answer_3 = qa.invoke("How are enforcement actions handled under the intercreditor agreements? Are there specific procedures for appointing an enforcement agent or for coordinating enforcement between creditors?")

    Intercreditor_answer_4 = qa.invoke("What are the standstill and turnover provisions in the intercreditor agreements? Under what circumstances can a creditor be prevented from taking enforcement actions?")

    Intercreditor_answer_5 = qa.invoke("How are payment blockages or restrictions handled between senior and junior creditors? What are the limitations or conditions imposed on junior creditors under these provisions?")

    Intercreditor_answer_6 = qa.invoke("What are the mechanisms for resolving disputes or conflicts between creditors in the intercreditor agreements?")

    Intercreditor_answer_7 = qa.invoke("How do the intercreditor agreements handle the distribution of proceeds in the event of enforcement or restructuring? What are the priority rules for distributing recoveries among creditors?")

    Intercreditor_answer_8 = qa.invoke("Describe the provisions related to amendments and waivers in the intercreditor agreements. How are decisions to amend the agreement or waive certain rights made among creditors?")

    Intercreditor_answer_9 = qa.invoke("What restrictions or limitations exist in the intercreditor agreements concerning the ability of junior creditors to exercise rights in insolvency or restructuring proceedings?")

    Intercreditor_answer_10 = qa.invoke("What reporting or information-sharing obligations exist under the intercreditor agreements? How frequently must creditors be updated, and what type of information is shared between creditor groups?")

    intercreditor_output = f"""**__Parties and Obligations under Intercreditor Agreements:__** {Intercreditor_answer_1} \n\n
    **__Ranking and Priority of Claims:__** {Intercreditor_answer_2} \n\n
    **__Enforcement Actions and Coordination:__** {Intercreditor_answer_3} \n\n
    **__Standstill and Turnover Provisions:__** {Intercreditor_answer_4} \n\n
    **__Payment Blockages and Restrictions:__** {Intercreditor_answer_5} \n\n
    **__Dispute Resolution Mechanisms:__** {Intercreditor_answer_6} \n\n
    **__Distribution of Proceeds and Priority Rules:__** {Intercreditor_answer_7} \n\n
    **__Amendments and Waivers:__** {Intercreditor_answer_8} \n\n
    **__Restrictions on Junior Creditors in Insolvency:__** {Intercreditor_answer_9} \n\n
    **__Information-Sharing Obligations:__** {Intercreditor_answer_10}"""
        
    with open("intercreditor_analysis.txt", 'w') as file:
        file.write(intercreditor_output)

    return intercreditor_output

#-------------
@st.cache_data
def generate_strategy() -> str:
    combined_output = ""
    
    # Check if there are files in the Corporate_Documents folder
    if company_document and os.path.exists("Corporate_Documents") and os.listdir("Corporate_Documents"):
        company_output = company_strategy()
        combined_output += company_output + '\n\n'

    # Check if there are files in the Financial_Documents folder
    if financial_document and os.path.exists("Financial_Documents") and os.listdir("Financial_Documents"):
        financial_output = financial_strategy()
        combined_output += financial_output + '\n\n'

    # Check if there are files in the Intercreditor_Documents folder
    if intercreditor_document and os.path.exists("Intercreditor_Documents") and os.listdir("Intercreditor_Documents"):
        intercreditor_output = intercreditor_strategy()
        combined_output += intercreditor_output + '\n\n'

    # Set the combined result in a single session state key
    st.session_state.results["legal_analysis_button_key"] = combined_output
    
    return combined_output
#---------------
#@st.cache_data
def create_pdf():
    company_content = ""
    financial_content = ""
    intercreditor_content = ""

    # Check if 'company_analysis.txt' exists and open if it does
    if os.path.exists('company_analysis.txt'):
        with open('company_analysis.txt', 'r') as file1:
            company_content = file1.read()

    # Check if 'financial_analysis.txt' exists and open if it does
    if os.path.exists('financial_analysis.txt'):
        with open('financial_analysis.txt', 'r') as file2:
            financial_content = file2.read()

    # Check if 'intercreditor_analysis.txt' exists and open if it does
    if os.path.exists('intercreditor_analysis.txt'):
        with open('intercreditor_analysis.txt', 'r') as file3:
            intercreditor_content = file3.read()

    # Combine the contents of the available files
    combined_content = ""

    if company_content:
        combined_content += company_content + '\n\n'

    if financial_content:
        combined_content += financial_content + '\n\n'

    if intercreditor_content:
        combined_content += intercreditor_content + '\n\n'

    # Write the combined content to a new file only if any content exists
    if combined_content:
        with open('legal_analysis.txt', 'w') as outfile:
            outfile.write(combined_content.strip()) 

    text_file = "legal_analysis.txt"
    pdf = FPDF('P', 'mm', 'A4')
    pdf.add_page()
    pdf.set_margins(10, 10, 10)
    pdf.set_font("Arial", size=15)

    pdf.cell(0, 10, txt="Structured Legal Analysis", ln=2, align='C')
    pdf.ln(5)

    pdf.set_font("Arial", size=11)
    try:
        with open(text_file, 'r', encoding='utf-8') as f:
            for line in f:
                pdf.multi_cell(0, 6, txt=line.encode('latin-1', 'replace').decode('latin-1'), align='L')
            pdf.ln(5)
    except UnicodeEncodeError:
        print("UnicodeEncodeError: Some characters could not be encoded in Latin-1. Skipping...")
        pass  # Skip the lines causing UnicodeEncodeError

    output_pdf_path = "ESG_analysis.pdf"
    pdf.output(output_pdf_path)

#----------------
#llm = build_llm()

if 'results' not in st.session_state:
    st.session_state.results = {
        "legal_analysis_button_key": {}
    }

loaders = {'.pdf': PyMuPDFLoader,
           '.xml': UnstructuredXMLLoader,
           '.csv': CSVLoader,
           }

def create_directory_loader(file_type, directory_path):
    return DirectoryLoader(
        path=directory_path,
        glob=f"**/*{file_type}",
        loader_cls=loaders[file_type],
    )


strategies_container = st.container()
with strategies_container:
    mrow1_col1, mrow1_col2 = st.columns(2)

    st.sidebar.info("To get started, please upload the documents from the company you would like to analyze.")
    button_container = st.sidebar.container()
    if os.path.exists("company_analysis.txt") and os.path.exists("financial_analysis.txt"):
        create_pdf()
        with open("ESG_analysis.pdf", "rb") as pdf_file:
            PDFbyte = pdf_file.read()

        st.sidebar.download_button(label="Download Analyses",
                    data=PDFbyte,
                    file_name="strategy_sheet.pdf",
                    mime='application/octet-stream',
                    )

    if button_container.button("Clear All"):
        
        st.session_state.button_states = {
        "legal_analysis_button_key": False,
        }
        st.session_state.button_states = {
        "financial_analysis_button_key": False,
        }
        st.session_state.results = {}

        st.session_state['history'] = []
        st.session_state['generated'] = ["Let's discuss the company documents 🤗"]
        st.session_state['past'] = ["Hey ! 👋"]
        st.cache_data.clear()
        st.cache_resource.clear()

        # Check if the subfolder exists
        if os.path.exists("Corporate_Documents"):
            for filename in os.listdir("Corporate_Documents"):
                file_path = os.path.join("Corporate_Documents", filename)
                try:
                    if os.path.isfile(file_path):
                        os.unlink(file_path)
                except Exception as e:
                    st.error(f"Error deleting {file_path}: {e}")
        else:
            pass

        if os.path.exists("Financial_Documents"):
            # Iterate through files in the subfolder and delete them
            for filename in os.listdir("Financial_Documents"):
                file_path = os.path.join("Financial_Documents", filename)
                try:
                    if os.path.isfile(file_path):
                        os.unlink(file_path)
                except Exception as e:
                    st.error(f"Error deleting {file_path}: {e}")
        else:
            pass
            # st.warning("No 'data' subfolder found.")
        
        if os.path.exists("Intercreditor_Documents"):
            # Iterate through files in the subfolder and delete them
            for filename in os.listdir("Intercreditor_Documents"):
                file_path = os.path.join("Intercreditor_Documents", filename)
                try:
                    if os.path.isfile(file_path):
                        os.unlink(file_path)
                except Exception as e:
                    st.error(f"Error deleting {file_path}: {e}")
        else:
            pass
            # st.warning("No 'data' subfolder found.")

        folders_to_clean = ["data", "chroma_db_portfolio", "chroma_db_LT", "chroma_db_fin"]

        for folder_path in folders_to_clean:
            if os.path.exists(folder_path):
                for item in os.listdir(folder_path):
                    item_path = os.path.join(folder_path, item)
                    try:
                        if os.path.isfile(item_path) or os.path.islink(item_path):
                            os.unlink(item_path)  # Remove files or symbolic links
                        elif os.path.isdir(item_path):
                            shutil.rmtree(item_path)  # Remove subfolders and all their contents
                    except Exception as e:
                        st.error(f"Error deleting {item_path}: {e}")
            else:
                pass
                # st.warning(f"No '{folder_path}' folder found.")

    with mrow1_col1:
        st.subheader("Legal Document Analysis")
        st.info("This tool is designed to provide a legal analysis of the documentation for  institutional investors.")
        
        button_container2 = st.container()
        if "button_states" not in st.session_state:
            st.session_state.button_states = {
            "legal_analysis_button_key": False,
            }
        
        if "results" not in st.session_state:
            st.session_state.results = {}

        if button_container2.button("Legal Analysis", key=legal_analysis_button_key):
            st.session_state.button_states[legal_analysis_button_key] = True
            result_generator = generate_strategy()  # Call the generator function
            st.session_state.results["legal_analysis_output"] = result_generator
            
        if "legal_analysis_output" in st.session_state.results:           
            st.write(st.session_state.results["legal_analysis_output"])
        st.divider()
        
    with mrow1_col2:
        if "legal_analysis_button_key" in st.session_state.results and st.session_state.results["legal_analysis_button_key"]:
            
            run_id = str(uuid.uuid4())

            scratchpad = {
                "questions": [],  # list of type Question
                "answerpad": [],
            }

            if company_document:
                store = create_vector_database_company()
            elif financial_document:
                store = create_vector_database_financial()
            elif intercreditor_document:
                store = create_vector_database_intercreditor()
            else:
                pass
                
            agent_settings = {
                "max_iterations": 3,
                "num_atomistic_questions": 2,
                "num_questions_per_iteration": 4,
                "question_atomizer_temperature": 0,
                "question_creation_temperature": 0.4,
                "question_prioritisation_temperature": 0,
                "refine_answer_temperature": 0,
                "qa_temperature": 0,
                "analyser_temperature": 0,
                "intermediate_answers_length": 200,
                "answer_length": 500,
            }

            # Updated prompt templates to include chat history
            def format_chat_history(chat_history):
                """Format chat history as a single string for input to the chain."""
                formatted_history = "\n".join([f"User: {entry['input']}\nAI: {entry['output']}" for entry in chat_history])
                return formatted_history

            # Initialize the agent with LCEL tools and memory
            memory = ConversationBufferMemory(memory_key="chat_history", k=3, return_messages=True)
            agent = Agent(agent_settings, scratchpad, store, True)
            def conversational_chat(query):
                # Get the result from the agent
                agent.run({"input": query, "chat_history": st.session_state['history']})
                
                result = agent.get_latest_answer()

                # Handle different response types
                if isinstance(result, dict):
                    # Extract the main content if the result is a dictionary
                    result = result.get("output", "")  # Adjust the key as needed based on your agent's output
                elif isinstance(result, list):
                    # If the result is a list, join it into a single string
                    result = "\n".join(result)
                elif not isinstance(result, str):
                    # Convert the result to a string if it is not already one
                    result = str(result)
                
                # Add the query and the result to the session state
                st.session_state['history'].append((query, result))
                
                # Update memory with the conversation
                memory.save_context({"input": query}, {"output": result})
                
                # Return the result
                return result

            # Ensure session states are initialized
            if 'history' not in st.session_state:
                st.session_state['history'] = []

            if 'generated' not in st.session_state:
                st.session_state['generated'] = ["Let's discuss the legal and financial matters 🤗"]

            if 'past' not in st.session_state:
                st.session_state['past'] = ["Hey ! 👋"]

            if 'input' not in st.session_state:
                st.session_state['input'] = ""

            # Streamlit layout
            st.subheader("Discuss the documentation")
            st.info("This tool is designed to enable discussion about the company's corporate and financial documentation.")
            response_container = st.container()
            container = st.container()

            with container:
                with st.form(key='my_form'):
                    user_input = st.text_input("Query:", placeholder="What would you like to know about the documentation", key='input')
                    submit_button = st.form_submit_button(label='Send')
                if submit_button and user_input:
                    output = conversational_chat(user_input)
                    st.session_state['past'].append(user_input)
                    st.session_state['generated'].append(output)
                    user_input = "Query:"
                #st.session_state['input'] = ""
            # Display generated responses
            if st.session_state['generated']:
                with response_container:
                    for i in range(len(st.session_state['generated'])):
                        message(st.session_state["past"][i], is_user=True, key=str(i) + '_user', avatar_style="shapes")
                        message(st.session_state["generated"][i], key=str(i), avatar_style="icons")