Upload 24 files

First app version

AM_Document_Analysis_v1.1.py

@@ -0,0 +1,1151 @@
+import os
+import shutil
+import streamlit as st
+from fpdf import FPDF
+from chromadb import Client
+from chromadb.config import Settings
+import json
+import chromadb
+from langchain_community.utilities import SerpAPIWrapper
+from llama_index.core import VectorStoreIndex
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from langchain_groq import ChatGroq
+from langchain.chains import LLMChain
+from langchain.agents import AgentType, Tool, initialize_agent, AgentExecutor
+from llama_parse import LlamaParse
+from langchain_community.document_loaders import UnstructuredMarkdownLoader
+from langchain_huggingface import HuggingFaceEmbeddings
+from llama_index.core import SimpleDirectoryReader
+from dotenv import load_dotenv, find_dotenv
+from streamlit_chat import message
+from langchain_community.vectorstores import Chroma
+from langchain_community.utilities import SerpAPIWrapper
+from langchain.chains import RetrievalQA
+from langchain_community.document_loaders import DirectoryLoader
+from langchain_community.document_loaders import PyMuPDFLoader
+from langchain_community.document_loaders import UnstructuredXMLLoader
+from langchain_community.document_loaders import CSVLoader
+from langchain.prompts import PromptTemplate
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import PromptTemplate
+import joblib
+import nltk
+from dotenv import load_dotenv, find_dotenv
+import uuid
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import PromptTemplate
+from yachalk import chalk
+from langchain.vectorstores import PGVector
+from langchain.document_loaders import PyPDFLoader, UnstructuredPDFLoader, PyPDFium2Loader
+from langchain.document_loaders import PyPDFDirectoryLoader
+## Import all the chains.
+from chains_v2.create_questions import QuestionCreationChain
+from chains_v2.most_pertinent_question import MostPertinentQuestion
+from chains_v2.retrieval_qa import retrieval_qa
+from chains_v2.research_compiler import research_compiler
+from chains_v2.question_atomizer import QuestionAtomizer
+from chains_v2.refine_answer import RefineAnswer
+## Import all the helpers.
+from helpers.response_helpers import result2QuestionsList
+from helpers.response_helpers import qStr2Dict
+from helpers.questions_helper import getAnsweredQuestions
+from helpers.questions_helper import getUnansweredQuestions
+from helpers.questions_helper import getSubQuestions
+from helpers.questions_helper import getHopQuestions
+from helpers.questions_helper import getLastQuestionId
+from helpers.questions_helper import markAnswered
+from helpers.questions_helper import getQuestionById
+
+import nest_asyncio  # noqa: E402
+nest_asyncio.apply()
+
+load_dotenv()
+load_dotenv(find_dotenv())
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+SERPAPI_API_KEY = os.environ["SERPAPI_API_KEY"]
+GOOGLE_CSE_ID = os.environ["GOOGLE_CSE_ID"]
+GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"]
+LLAMA_PARSE_API_KEY = os.environ["LLAMA_PARSE_API_KEY"]
+HUGGINGFACEHUB_API_TOKEN = os.environ["HUGGINGFACEHUB_API_TOKEN"]
+LANGCHAIN_API_KEY = os.environ["LANGCHAIN_API_KEY"]
+LANGCHAIN_ENDPOINT = os.environ["LANGCHAIN_ENDPOINT"]
+LANGCHAIN_PROJECT = os.environ["LANGCHAIN_PROJECT"]
+groq_api_key=os.getenv('GROQ_API_KEY')
+
+st.set_page_config(layout="wide")
+
+css = """
+<style>
+    [data-testid="stAppViewContainer"] {
+        background-color: #f8f9fa; /* Very light grey */
+    }
+    [data-testid="stSidebar"] {
+        background-color: white;
+        color: black;
+    }
+    [data-testid="stAppViewContainer"] * {
+        color: black; /* Ensure all text is black */
+    }
+    button {
+        background-color: #add8e6; /* Light blue for primary buttons */
+        color: black;
+        border: 2px solid green; /* Green border */
+    }
+    button:hover {
+        background-color: #87ceeb; /* Slightly darker blue on hover */
+    }
+
+    button:active {
+        outline: 2px solid green; /* Green outline when the button is pressed */
+        outline-offset: 2px; /* Space between button and outline */
+    }
+
+    .stButton>button:first-child {
+        background-color: #add8e6; /* Light blue for primary buttons */
+        color: black;
+    }
+    .stButton>button:first-child:hover {
+        background-color: #87ceeb; /* Slightly darker blue on hover */
+    }
+    .stButton>button:nth-child(2) {
+        background-color: #b0e0e6; /* Even lighter blue for secondary buttons */
+        color: black;
+    }
+    .stButton>button:nth-child(2):hover {
+        background-color: #add8e6; /* Slightly darker blue on hover */
+    }
+    [data-testid="stFileUploadDropzone"] {
+        background-color: white; /* White background for file upload */
+    }
+    [data-testid="stFileUploadDropzone"] .stDropzone, [data-testid="stFileUploadDropzone"] .stDropzone input {
+        color: black; /* Ensure file upload text is black */
+    }
+    
+    .stButton>button:active {
+        outline: 2px solid green; /* Green outline when the button is pressed */
+        outline-offset: 2px;
+    }
+</style>
+"""
+def load_credentials(filepath):
+    with open(filepath, 'r') as file:
+        return json.load(file)
+
+# Load credentials from 'credentials.json'
+credentials = load_credentials('Assets/credentials.json')
+
+# Initialize session state if not already done
+if 'logged_in' not in st.session_state:
+    st.session_state.logged_in = False
+    st.session_state.username = ''
+
+# Function to handle login
+def login(username, password):
+    if username in credentials and credentials[username] == password:
+        st.session_state.logged_in = True
+        st.session_state.username = username
+        st.rerun()  # Rerun to reflect login state
+    else:
+        st.session_state.logged_in = False
+        st.session_state.username = ''
+        st.error("Invalid username or password.")
+
+# Function to handle logout
+def logout():
+    st.session_state.logged_in = False
+    st.session_state.username = ''
+    st.rerun()  # Rerun to reflect logout state
+
+#--------------
+## Define log printers
+
+def print_iteration(current_iteration):
+    print(
+        chalk.bg_yellow_bright.black.bold(
+            f"\n   Iteration - {current_iteration}  ▷▶  \n"
+        )
+    )
+
+
+def print_unanswered_questions(unanswered):
+    print(
+        chalk.cyan_bright("** Unanswered Questions **"),
+        chalk.cyan("".join([f"\n'{q['id']}. {q['question']}'" for q in unanswered])),
+    )
+
+
+def print_next_question(current_question_id, current_question):
+    print(
+        chalk.magenta.bold("** 🤔 Next Questions I must ask: **\n"),
+        chalk.magenta(current_question_id),
+        chalk.magenta(current_question["question"]),
+    )
+
+
+def print_answer(current_question):
+    print(
+        chalk.yellow_bright.bold("** Answer **\n"),
+        chalk.yellow_bright(current_question["answer"]),
+    )
+
+
+def print_final_answer(answerpad):
+    print(
+        chalk.white("** Refined Answer **\n"),
+        chalk.white(answerpad[-1]),
+    )
+
+
+def print_max_iterations():
+    print(
+        chalk.bg_yellow_bright.black.bold(
+            "\n ✔✔  Max Iterations Reached. Compiling the results ...\n"
+        )
+    )
+
+
+def print_result(result):
+    print(chalk.italic.white_bright((result["text"])))
+
+
+def print_sub_question(q):
+    print(chalk.magenta.bold(f"** Sub Question **\n{q['question']}\n{q['answer']}\n"))
+#--------------
+## ---- The researcher ----- ##
+
+class Agent:
+    ## Create chains
+    def __init__(self, agent_settings, scratchpad, store, verbose):
+        self.store = store
+        self.scratchpad = scratchpad
+        self.agent_settings = agent_settings
+        self.verbose = verbose
+        self.question_creation_chain = QuestionCreationChain.from_llm(
+            language_model(
+                temperature=self.agent_settings["question_creation_temperature"]
+            ),
+            verbose=self.verbose,
+        )
+        self.question_atomizer = QuestionAtomizer.from_llm(
+            llm=language_model(
+                temperature=self.agent_settings["question_atomizer_temperature"]
+            ),
+            verbose=self.verbose,
+        )
+        self.most_pertinent_question = MostPertinentQuestion.from_llm(
+            language_model(
+                temperature=self.agent_settings["question_creation_temperature"]
+            ),
+            verbose=self.verbose,
+        )
+        self.refine_answer = RefineAnswer.from_llm(
+            language_model(
+                temperature=self.agent_settings["refine_answer_temperature"]
+            ),
+            verbose=self.verbose,
+        )
+
+    def run(self, question):
+        ## Step 0. Prepare the initial set of questions
+        atomized_questions_response = self.question_atomizer.run(
+            question=question,
+            num_questions=self.agent_settings["num_atomistic_questions"],
+        )
+
+        self.scratchpad["questions"] += result2QuestionsList(
+            question_response=atomized_questions_response,
+            type="subquestion",
+            status="unanswered",
+        )
+
+        for q in self.scratchpad["questions"]:
+            q["answer"], q["documents"] = retrieval_qa(
+                llm=language_model(
+                    temperature=self.agent_settings["qa_temperature"],
+                    verbose=self.verbose,
+                ),
+                retriever=self.store.as_retriever(
+                    search_type="mmr", search_kwargs={"k": 5, "fetch_k": 10}
+                ),
+                question=q["question"],
+                answer_length=self.agent_settings["intermediate_answers_length"],
+                verbose=self.verbose,
+            )
+            q["status"] = "answered"
+            print_sub_question(q)
+
+        
+        current_context = "".join(
+            f"\n{q['id']}. {q['question']}\n{q['answer']}\n"
+            for q in self.scratchpad["questions"]
+        )
+        
+        self.scratchpad["answerpad"] += [current_context]
+
+        current_iteration = 0
+
+        while True:
+            current_iteration += 1
+            print_iteration(current_iteration)
+
+            # STEP 1: create questions
+            start_id = getLastQuestionId(self.scratchpad["questions"]) + 1
+            questions_response = self.question_creation_chain.run(
+                question=question,
+                context=current_context,
+                previous_questions=[
+                    "".join(f"\n{q['question']}") for q in self.scratchpad["questions"]
+                ],
+                num_questions=self.agent_settings["num_questions_per_iteration"],
+                start_id=start_id,
+            )
+            self.scratchpad["questions"] += result2QuestionsList(
+                question_response=questions_response,
+                type="hop",
+                status="unanswered",
+            )
+
+            # STEP 2: Choose question for current iteration
+            unanswered = getUnansweredQuestions(self.scratchpad["questions"])
+            unanswered_questions_prompt = self.unanswered_questions_prompt(unanswered)
+            print_unanswered_questions(unanswered)
+            response = self.most_pertinent_question.run(
+                original_question=question,
+                unanswered_questions=unanswered_questions_prompt,
+            )
+            current_question_dict = qStr2Dict(question=response)
+            current_question_id = current_question_dict["id"]
+            current_question = getQuestionById(
+                self.scratchpad["questions"], current_question_id
+            )
+            print_next_question(current_question_id, current_question)
+
+            # STEP 3: Answer the question
+            current_question["answer"], current_question["documents"] = retrieval_qa(
+                llm=language_model(
+                    temperature=self.agent_settings["qa_temperature"],
+                    verbose=self.verbose,
+                ),
+                retriever=self.store.as_retriever(
+                    search_type="mmr", search_kwargs={"k": 5, "fetch_k": 10}
+                ),
+                question=current_question["question"],
+                answer_length=self.agent_settings["intermediate_answers_length"],
+                verbose=self.verbose,
+            )
+            markAnswered(self.scratchpad["questions"], current_question_id)
+            print_answer(current_question)
+            current_context = current_question["answer"]
+
+            ## STEP 4: refine the answer
+            refinement_context = current_question["question"] + "\n" + current_context
+            refine_answer = self.refine_answer.run(
+                question=question,
+                context=refinement_context,
+                answer=self.get_latest_answer(),
+            )
+            self.scratchpad["answerpad"] += [refine_answer]
+            print_final_answer(self.scratchpad["answerpad"])
+
+            if current_iteration > self.agent_settings["max_iterations"]:
+                print_max_iterations()
+                break
+
+    def unanswered_questions_prompt(self, unanswered):
+        return (
+            "[" + "".join([f"\n{q['id']}. {q['question']}" for q in unanswered]) + "]"
+        )
+
+    def notes_prompt(self, answered_questions):
+        return "".join(
+            [
+                f"{{ Question: {q['question']}, Answer: {q['answer']} }}"
+                for q in answered_questions
+            ]
+        )
+
+    def get_latest_answer(self):
+        answers = self.scratchpad["answerpad"]
+        answer = answers[-1] if answers else ""
+        return answer
+    
+#--------------
+# If not logged in, show login form
+if not st.session_state.logged_in:
+    st.sidebar.write("Login")
+    username = st.sidebar.text_input('Username')
+    password = st.sidebar.text_input('Password', type='password')
+    if st.sidebar.button('Login'):
+        login(username, password)
+    # Stop the script here if the user is not logged in
+    st.stop()
+
+
+# If logged in, show logout button and main content
+st.sidebar.image('StratXcel.png', width=150)
+if st.session_state.logged_in:
+    st.sidebar.write(f"Welcome, {st.session_state.username}!")
+    if st.sidebar.button('Logout'):
+        logout()
+
+st.write(css, unsafe_allow_html=True)
+
+company_document = st.sidebar.toggle("Company document", False)
+financial_document = st.sidebar.toggle("Financial document", False)
+intercreditor_document = st.sidebar.toggle("Intercreditor document", False)
+
+#------------- 
+llm=ChatGroq(groq_api_key=groq_api_key,
+             model_name="Llama-3.1-70b-Versatile", temperature = 0.0, streaming=True)   
+
+Llama = "Llama-3.1-70b-Versatile"
+def language_model(
+    model_name: str = Llama, temperature: float = 0, verbose: bool = False
+    ):
+    llm=ChatGroq(groq_api_key=groq_api_key, model_name=model_name, temperature=temperature, verbose=verbose)    
+    return llm
+#--------------
+doc_retriever_company = None
+doc_retriever_financials = None
+doc_retriever_intercreditor = None
+
+#--------------
+
+#@st.cache_data
+def load_or_parse_data_company():
+    data_file = "./data/parsed_data_company.pkl"
+
+    parsingInstructionUber10k = """The provided documents are company law documents of a company.
+    They contain detailed information about the rights and obligations of the company and its shareholders and contracting parties.
+    They also contain procedures for dispute resolution, voting, control priority and exit and sale situations. 
+    You must never provide false legal or financial information. Use only the information included in the context documents.
+    Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""
+
+    parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
+                        result_type="markdown",
+                        parsing_instruction=parsingInstructionUber10k,
+                        max_timeout=5000,
+                        gpt4o_mode=True,
+                        )
+
+    file_extractor = {".pdf": parser}
+    reader = SimpleDirectoryReader("./Corporate_Documents", file_extractor=file_extractor)
+    documents = reader.load_data()
+
+    print("Saving the parse results in .pkl format ..........")
+    joblib.dump(documents, data_file)
+
+    # Set the parsed data to the variable
+    parsed_data_company = documents
+
+    return parsed_data_company
+
+#@st.cache_data
+def load_or_parse_data_financial():
+    data_file = "./data/parsed_data_financial.pkl"
+
+    parsingInstructionUber10k = """The provided documents are financial law documents of a company.
+    They contain detailed information about the rights and obligations of the company and its creditors.
+    They also contain procedures for acceleration of debt, sale of security, enforcement, use of creditor control, priority and distribution of assets. 
+    You must never provide false legal or financial information. Use only the information included in the context documents.
+    Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""
+
+    parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
+                        result_type="markdown",
+                        parsing_instruction=parsingInstructionUber10k,
+                        max_timeout=5000,
+                        gpt4o_mode=True,
+                        )
+
+    file_extractor = {".pdf": parser}
+    reader = SimpleDirectoryReader("./Financial_Documents", file_extractor=file_extractor)
+    documents = reader.load_data()
+
+    print("Saving the parse results in .pkl format ..........")
+    joblib.dump(documents, data_file)
+
+    # Set the parsed data to the variable
+    parsed_data_financial = documents
+
+    return parsed_data_financial
+
+#--------------
+
+#@st.cache_data
+def load_or_parse_data_intercreditor():
+    data_file = "./data/parsed_data_intercreditor.pkl"
+
+    parsingInstructionUber10k = """The provided documents are intercreditor agreements a company .
+    They contain detailed information about the rights and obligations of the company and its creditors and creditor groups.
+    They also contain procedures for acceleration of debt, sale of security, enforcement, use of creditor control, priority and distribution of assets. 
+    You must never provide false legal or financial information. Use only the information included in the context documents.
+    Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""
+
+    parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
+                        result_type="markdown",
+                        parsing_instruction=parsingInstructionUber10k,
+                        max_timeout=5000,
+                        gpt4o_mode=True,
+                        )
+
+    file_extractor = {".pdf": parser}
+    reader = SimpleDirectoryReader("./Intercreditor_Documents", file_extractor=file_extractor)
+    documents = reader.load_data()
+
+    print("Saving the parse results in .pkl format ..........")
+    joblib.dump(documents, data_file)
+
+    # Set the parsed data to the variable
+    parsed_data_financial = documents
+
+    return parsed_data_financial
+#--------------
+# Create vector database
+
+@st.cache_resource
+def create_vector_database_company():
+    # Call the function to either load or parse the data
+    llama_parse_documents = load_or_parse_data_company()
+
+    with open('data/output_company.md', 'a') as f:  # Open the file in append mode ('a')
+        for doc in llama_parse_documents:
+            f.write(doc.text + '\n')
+
+    markdown_path = "data/output_company.md"
+    loader = UnstructuredMarkdownLoader(markdown_path)
+    documents = loader.load()
+    # Split loaded documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=30)
+    docs = text_splitter.split_documents(documents)
+
+    #len(docs)
+    print(f"length of documents loaded: {len(documents)}")
+    print(f"total number of document chunks generated :{len(docs)}")
+    embed_model = HuggingFaceEmbeddings()
+    print('Vector DB not yet created !')
+
+    vs = Chroma.from_documents(
+        documents=docs,
+        embedding=embed_model,
+        collection_name="rag",
+    )
+
+    doc_retriever_company = vs
+    #doc_retriever_company = vs.as_retriever()
+    
+    print('Vector DB created successfully !')
+    return doc_retriever_company
+
+@st.cache_resource
+def create_vector_database_financial():
+    # Call the function to either load or parse the data
+    llama_parse_documents = load_or_parse_data_financial()
+
+    with open('data/output_financials.md', 'a') as f:  # Open the file in append mode ('a')
+        for doc in llama_parse_documents:
+            f.write(doc.text + '\n')
+
+    markdown_path = "data/output_financials.md"
+    loader = UnstructuredMarkdownLoader(markdown_path)
+    documents = loader.load()
+    # Split loaded documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15)
+    docs = text_splitter.split_documents(documents)
+
+    print(f"length of documents loaded: {len(documents)}")
+    print(f"total number of document chunks generated :{len(docs)}")
+    embed_model = HuggingFaceEmbeddings()
+
+    vs = Chroma.from_documents(
+        documents=docs,
+        embedding=embed_model,
+        collection_name="rag"
+    )
+    doc_retriever_financial = vs
+    #doc_retriever_financial = vs.as_retriever()
+
+    print('Vector DB created successfully !')
+    return doc_retriever_financial
+
+#--------------
+
+@st.cache_resource
+def create_vector_database_intercreditor():
+    # Call the function to either load or parse the data
+    llama_parse_documents = load_or_parse_data_intercreditor()
+
+    with open('data/output_intercreditor.md', 'a') as f:  # Open the file in append mode ('a')
+        for doc in llama_parse_documents:
+            f.write(doc.text + '\n')
+
+    markdown_path = "data/output_intercreditor.md"
+    loader = UnstructuredMarkdownLoader(markdown_path)
+    documents = loader.load()
+    # Split loaded documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15)
+    docs = text_splitter.split_documents(documents)
+
+    print(f"length of documents loaded: {len(documents)}")
+    print(f"total number of document chunks generated :{len(docs)}")
+    embed_model = HuggingFaceEmbeddings()
+
+    vs = Chroma.from_documents(
+        documents=docs,
+        embedding=embed_model,
+        collection_name="rag"
+    )
+    doc_retriever_intercreditor = vs
+    #doc_retriever_intercreditor = vs.as_retriever()
+
+    print('Vector DB created successfully !')
+    return doc_retriever_intercreditor
+
+#--------------
+legal_analysis_button_key = "legal_strategy_button"
+
+#---------------
+def delete_files_and_folders(folder_path):
+    for root, dirs, files in os.walk(folder_path, topdown=False):
+        for file in files:
+            try:
+                os.unlink(os.path.join(root, file))
+            except Exception as e:
+                st.error(f"Error deleting {os.path.join(root, file)}: {e}")
+        for dir in dirs:
+            try:
+                os.rmdir(os.path.join(root, dir))
+            except Exception as e:
+                st.error(f"Error deleting directory {os.path.join(root, dir)}: {e}")
+#---------------
+
+if company_document:
+    uploaded_files_ESG = st.sidebar.file_uploader("Choose company law documents", accept_multiple_files=True, key="company_files")
+    for uploaded_file in uploaded_files_ESG:
+        st.write("filename:", uploaded_file.name)
+        def save_uploadedfile(uploadedfile):
+            with open(os.path.join("Corporate_Documents",uploadedfile.name),"wb") as f:
+                f.write(uploadedfile.getbuffer())
+            return st.success("Saved File:{} to Company_Documents".format(uploadedfile.name))
+        save_uploadedfile(uploaded_file)
+
+if financial_document:
+    uploaded_files_financials = st.sidebar.file_uploader("Choose financial law documents", accept_multiple_files=True, key="financial_files")
+    for uploaded_file in uploaded_files_financials:
+        st.write("filename:", uploaded_file.name)
+        def save_uploadedfile(uploadedfile):
+            with open(os.path.join("Financial_Documents",uploadedfile.name),"wb") as f:
+                f.write(uploadedfile.getbuffer())
+            return st.success("Saved File:{} to Financial_Documents".format(uploadedfile.name))
+        save_uploadedfile(uploaded_file)
+
+if intercreditor_document:
+    uploaded_files_intercreditor = st.sidebar.file_uploader("Choose intercreditor documents", accept_multiple_files=True, key="intercreditor_files")
+    for uploaded_file in uploaded_files_intercreditor:
+        st.write("filename:", uploaded_file.name)
+        def save_uploadedfile(uploadedfile):
+            with open(os.path.join("Intercreditor_Documents",uploadedfile.name),"wb") as f:
+                f.write(uploadedfile.getbuffer())
+            return st.success("Saved File:{} to Intercreditor_Documents".format(uploadedfile.name))
+        save_uploadedfile(uploaded_file)
+#---------------
+def company_strategy():
+    doc_retriever_company = create_vector_database_company().as_retriever()
+    prompt_template = """<|system|>
+    You are a seasoned attorney specilizing in company and corporate law and legal analysis. You write expert analyses for institutional investors. 
+    Output must have sub-headings in bold font and be fluent.<|end|>
+    <|user|>
+    Answer the {question} based on the information you find in context: {context} <|end|>
+    <|assistant|>""" 
+
+    prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])
+
+    qa = (
+    {
+        "context": doc_retriever_company,
+        "question": RunnablePassthrough(),
+    }
+    | prompt
+    | llm
+    | StrOutputParser()
+)   
+
+    Corporate_answer_1 = qa.invoke("What provisions govern the appointment and removal of directors in the company? Outline the procedures and any required shareholder involvement in these processes.")
+
+    Corporate_answer_2 = qa.invoke("Explain the company's share capital structure, including any provisions for different classes of shares and the rights attached to them. How are voting rights distributed among shareholders?")
+
+    Corporate_answer_3 = qa.invoke("What restrictions or conditions are placed on the transfer or sale of shares in the company's articles of association or shareholders' agreements? Include any pre-emption rights or lock-in provisions.")
+
+    Corporate_answer_4 = qa.invoke("Describe the rights and obligations of majority and minority shareholders as outlined in the company's shareholders' agreements. What protections are in place for minority shareholders?")
+
+    Corporate_answer_5 = qa.invoke("What are the provisions for issuing new shares or increasing the company's capital? Detail any existing shareholder approval requirements or pre-emptive rights outlined in the company's governing documents.")
+
+    Corporate_answer_6 = qa.invoke("Outline the procedures for decision-making in shareholder meetings, including quorum requirements and voting thresholds for ordinary and special resolutions. How are dissenting shareholders addressed in key decisions?")
+
+    Corporate_answer_7 = qa.invoke("What mechanisms are in place for resolving shareholder disputes? Provide details on any arbitration or mediation clauses found in the company's articles or shareholders' agreements.")
+
+    Corporate_answer_8 = qa.invoke("Describe the exit mechanisms available for shareholders, such as drag-along and tag-along rights, and the circumstances under which they can be triggered.")
+
+    Corporate_answer_9 = qa.invoke("What rights do shareholders have to appoint or remove members of the board? Outline any requirements for shareholder approval in relation to board appointments or dismissals.")
+
+    Corporate_answer_10 = qa.invoke("Explain any restrictions on the powers of the board as set out in the company's governing documents. Are there specific decisions that require shareholder approval or consultation?")
+
+    Corporate_answer_11 = qa.invoke("What provisions are in place regarding dividends and the distribution of profits? How are dividend rights structured among different classes of shares, if applicable?")
+
+
+    corporate_output = f"""**__Director Appointment and Removal:__** {Corporate_answer_1} \n\n 
+**__Share Capital Structure and Voting Rights:__** {Corporate_answer_2} \n\n 
+**__Restrictions on Share Transfer and Sale:__** {Corporate_answer_3} \n\n 
+**__Rights and Obligations of Shareholders:__** {Corporate_answer_4} \n\n 
+**__Issuing New Shares and Capital Increases:__** {Corporate_answer_5} \n\n 
+**__Decision-Making in Shareholder Meetings:__** {Corporate_answer_6} \n\n 
+**__Shareholder Dispute Resolution Mechanisms:__** {Corporate_answer_7} \n\n 
+**__Exit Mechanisms for Shareholders:__** {Corporate_answer_8} \n\n 
+**__Rights to Appoint or Remove Board Members:__** {Corporate_answer_9} \n\n 
+**__Restrictions on the Board's Powers:__** {Corporate_answer_10} \n\n 
+**__Dividend and Profit Distribution Provisions:__** {Corporate_answer_11}"""
+    
+    financial_output = corporate_output
+    
+    with open("company_analysis.txt", 'w') as file:
+        file.write(financial_output)
+    
+    return financial_output
+
+def financial_strategy():
+    doc_retriever_financial = create_vector_database_financial().as_retriever()
+    prompt_template = """<|system|>
+    You are a seasoned attorney specilizing in financial law and legal analysis. You write expert analyses for institutional investors. 
+    Output must have sub-headings in bold font and be fluent.<|end|>
+    <|user|>
+    Answer the {question} based on the information you find in context: {context} <|end|>
+    <|assistant|>""" 
+
+    prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])
+
+    qa = (
+    {
+        "context": doc_retriever_financial,
+        "question": RunnablePassthrough(),
+    }
+    | prompt
+    | llm
+    | StrOutputParser()
+)   
+
+    Financial_answer_1 = qa.invoke("What are the parties of the agreements and key obligations of the borrower under the company's loan agreements? Describe any covenants or financial ratios the borrower must comply with.")
+
+    Financial_answer_3 = qa.invoke("What provisions govern the occurrence of events of default under the company's loan and bond agreements? Include any cross-default or material adverse change clauses.")
+
+    Financial_answer_4 = qa.invoke("Describe the rights of secured creditors under the company's security documents. What types of assets are secured, and how can creditors enforce their security in case of default?")
+
+    Financial_answer_5 = qa.invoke("What are the acceleration clauses in the company's financial agreements? Under what conditions can creditors demand early repayment or terminate financing arrangements?")
+
+    Financial_answer_6 = qa.invoke("Outline the procedures and requirements for enforcing security interests under the company's security documents. How do the rights of secured and unsecured creditors differ in this context?")
+
+    Financial_answer_7 = qa.invoke("How are decisions related to enforcement or restructuring prioritized among creditors?")
+
+    Financial_answer_8 = qa.invoke("Explain the company's obligations under any guarantees or indemnities provided to creditors. What are the limitations, if any, on the enforcement of these guarantees?")
+
+    Financial_answer_9 = qa.invoke("Describe the rights of bondholders or lenders in the company's bond issuance agreements or loans. What are the procedures for creditor meetings, and how can creditors exercise their rights in the event of default?")
+
+    Financial_answer_10 = qa.invoke("What protections are in place for junior creditors or subordinated debt holders in the company's financial agreements? How are their rights affected in the event of enforcement or restructuring?")
+
+    Financial_answer_11 = qa.invoke("What are the company's obligations to provide financial information to creditors under its loan or bond agreements? How frequently must the company report, and what information is typically required?")
+
+
+    financial_output = f"""**__Borrower Obligations and Covenants:__** {Financial_answer_1} \n\n 
+**__Events of Default and Cross-Default Provisions:__** {Financial_answer_3} \n\n 
+**__Rights of Secured Creditors and Enforcement of Security:__** {Financial_answer_4} \n\n 
+**__Acceleration Clauses and Early Repayment Triggers:__** {Financial_answer_5} \n\n 
+**__Enforcement of Security Interests:__** {Financial_answer_6} \n\n 
+**__Intercreditor Decision-Making and Control:__** {Financial_answer_7} \n\n 
+**__Guarantees and Indemnities Obligations:__** {Financial_answer_8} \n\n 
+**__Rights of Bondholders and Default Procedures:__** {Financial_answer_9} \n\n 
+**__Protections for Junior Creditors:__** {Financial_answer_10} \n\n 
+**__Financial Reporting Obligations to Creditors:__** {Financial_answer_11}"""
+        
+    with open("financial_analysis.txt", 'w') as file:
+        file.write(financial_output)
+
+    return financial_output
+
+def intercreditor_strategy():
+    doc_retriever_intercreditor = create_vector_database_intercreditor().as_retriever()
+    prompt_template = """<|system|>
+    You are a seasoned attorney specilizing in financial law and legal analysis. You write expert analyses for institutional investors. 
+    Output must have sub-headings in bold font and be fluent.<|end|>
+    <|user|>
+    Answer the {question} based on the information you find in context: {context} <|end|>
+    <|assistant|>""" 
+
+    prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])
+
+    qa = (
+    {
+        "context": doc_retriever_intercreditor,
+        "question": RunnablePassthrough(),
+    }
+    | prompt
+    | llm
+    | StrOutputParser()
+)   
+
+    Intercreditor_answer_1 = qa.invoke("What are the parties involved in the intercreditor agreements, and what are their key roles and obligations?")
+
+    Intercreditor_answer_2 = qa.invoke("What provisions govern the ranking and priority of claims among creditors under the intercreditor agreements? Describe any subordination terms or waterfall clauses.")
+
+    Intercreditor_answer_3 = qa.invoke("How are enforcement actions handled under the intercreditor agreements? Are there specific procedures for appointing an enforcement agent or for coordinating enforcement between creditors?")
+
+    Intercreditor_answer_4 = qa.invoke("What are the standstill and turnover provisions in the intercreditor agreements? Under what circumstances can a creditor be prevented from taking enforcement actions?")
+
+    Intercreditor_answer_5 = qa.invoke("How are payment blockages or restrictions handled between senior and junior creditors? What are the limitations or conditions imposed on junior creditors under these provisions?")
+
+    Intercreditor_answer_6 = qa.invoke("What are the mechanisms for resolving disputes or conflicts between creditors in the intercreditor agreements?")
+
+    Intercreditor_answer_7 = qa.invoke("How do the intercreditor agreements handle the distribution of proceeds in the event of enforcement or restructuring? What are the priority rules for distributing recoveries among creditors?")
+
+    Intercreditor_answer_8 = qa.invoke("Describe the provisions related to amendments and waivers in the intercreditor agreements. How are decisions to amend the agreement or waive certain rights made among creditors?")
+
+    Intercreditor_answer_9 = qa.invoke("What restrictions or limitations exist in the intercreditor agreements concerning the ability of junior creditors to exercise rights in insolvency or restructuring proceedings?")
+
+    Intercreditor_answer_10 = qa.invoke("What reporting or information-sharing obligations exist under the intercreditor agreements? How frequently must creditors be updated, and what type of information is shared between creditor groups?")
+
+    intercreditor_output = f"""**__Parties and Obligations under Intercreditor Agreements:__** {Intercreditor_answer_1} \n\n
+    **__Ranking and Priority of Claims:__** {Intercreditor_answer_2} \n\n
+    **__Enforcement Actions and Coordination:__** {Intercreditor_answer_3} \n\n
+    **__Standstill and Turnover Provisions:__** {Intercreditor_answer_4} \n\n
+    **__Payment Blockages and Restrictions:__** {Intercreditor_answer_5} \n\n
+    **__Dispute Resolution Mechanisms:__** {Intercreditor_answer_6} \n\n
+    **__Distribution of Proceeds and Priority Rules:__** {Intercreditor_answer_7} \n\n
+    **__Amendments and Waivers:__** {Intercreditor_answer_8} \n\n
+    **__Restrictions on Junior Creditors in Insolvency:__** {Intercreditor_answer_9} \n\n
+    **__Information-Sharing Obligations:__** {Intercreditor_answer_10}"""
+        
+    with open("intercreditor_analysis.txt", 'w') as file:
+        file.write(intercreditor_output)
+
+    return intercreditor_output
+
+#-------------
+@st.cache_data
+def generate_strategy() -> str:
+    combined_output = ""
+    
+    # Check if there are files in the Corporate_Documents folder
+    if company_document and os.path.exists("Corporate_Documents") and os.listdir("Corporate_Documents"):
+        company_output = company_strategy()
+        combined_output += company_output + '\n\n'
+
+    # Check if there are files in the Financial_Documents folder
+    if financial_document and os.path.exists("Financial_Documents") and os.listdir("Financial_Documents"):
+        financial_output = financial_strategy()
+        combined_output += financial_output + '\n\n'
+
+    # Check if there are files in the Intercreditor_Documents folder
+    if intercreditor_document and os.path.exists("Intercreditor_Documents") and os.listdir("Intercreditor_Documents"):
+        intercreditor_output = intercreditor_strategy()
+        combined_output += intercreditor_output + '\n\n'
+
+    # Set the combined result in a single session state key
+    st.session_state.results["legal_analysis_button_key"] = combined_output
+    
+    return combined_output
+#---------------
+#@st.cache_data
+def create_pdf():
+    company_content = ""
+    financial_content = ""
+    intercreditor_content = ""
+
+    # Check if 'company_analysis.txt' exists and open if it does
+    if os.path.exists('company_analysis.txt'):
+        with open('company_analysis.txt', 'r') as file1:
+            company_content = file1.read()
+
+    # Check if 'financial_analysis.txt' exists and open if it does
+    if os.path.exists('financial_analysis.txt'):
+        with open('financial_analysis.txt', 'r') as file2:
+            financial_content = file2.read()
+
+    # Check if 'intercreditor_analysis.txt' exists and open if it does
+    if os.path.exists('intercreditor_analysis.txt'):
+        with open('intercreditor_analysis.txt', 'r') as file3:
+            intercreditor_content = file3.read()
+
+    # Combine the contents of the available files
+    combined_content = ""
+
+    if company_content:
+        combined_content += company_content + '\n\n'
+
+    if financial_content:
+        combined_content += financial_content + '\n\n'
+
+    if intercreditor_content:
+        combined_content += intercreditor_content + '\n\n'
+
+    # Write the combined content to a new file only if any content exists
+    if combined_content:
+        with open('legal_analysis.txt', 'w') as outfile:
+            outfile.write(combined_content.strip()) 
+
+    text_file = "legal_analysis.txt"
+    pdf = FPDF('P', 'mm', 'A4')
+    pdf.add_page()
+    pdf.set_margins(10, 10, 10)
+    pdf.set_font("Arial", size=15)
+
+    pdf.cell(0, 10, txt="Structured Legal Analysis", ln=2, align='C')
+    pdf.ln(5)
+
+    pdf.set_font("Arial", size=11)
+    try:
+        with open(text_file, 'r', encoding='utf-8') as f:
+            for line in f:
+                pdf.multi_cell(0, 6, txt=line.encode('latin-1', 'replace').decode('latin-1'), align='L')
+            pdf.ln(5)
+    except UnicodeEncodeError:
+        print("UnicodeEncodeError: Some characters could not be encoded in Latin-1. Skipping...")
+        pass  # Skip the lines causing UnicodeEncodeError
+
+    output_pdf_path = "ESG_analysis.pdf"
+    pdf.output(output_pdf_path)
+
+#----------------
+#llm = build_llm()
+
+if 'results' not in st.session_state:
+    st.session_state.results = {
+        "legal_analysis_button_key": {}
+    }
+
+loaders = {'.pdf': PyMuPDFLoader,
+           '.xml': UnstructuredXMLLoader,
+           '.csv': CSVLoader,
+           }
+
+def create_directory_loader(file_type, directory_path):
+    return DirectoryLoader(
+        path=directory_path,
+        glob=f"**/*{file_type}",
+        loader_cls=loaders[file_type],
+    )
+
+
+strategies_container = st.container()
+with strategies_container:
+    mrow1_col1, mrow1_col2 = st.columns(2)
+
+    st.sidebar.info("To get started, please upload the documents from the company you would like to analyze.")
+    button_container = st.sidebar.container()
+    if os.path.exists("company_analysis.txt") and os.path.exists("financial_analysis.txt"):
+        create_pdf()
+        with open("ESG_analysis.pdf", "rb") as pdf_file:
+            PDFbyte = pdf_file.read()
+
+        st.sidebar.download_button(label="Download Analyses",
+                    data=PDFbyte,
+                    file_name="strategy_sheet.pdf",
+                    mime='application/octet-stream',
+                    )
+
+    if button_container.button("Clear All"):
+        
+        st.session_state.button_states = {
+        "legal_analysis_button_key": False,
+        }
+        st.session_state.button_states = {
+        "financial_analysis_button_key": False,
+        }
+        st.session_state.results = {}
+
+        st.session_state['history'] = []
+        st.session_state['generated'] = ["Let's discuss the company documents 🤗"]
+        st.session_state['past'] = ["Hey ! 👋"]
+        st.cache_data.clear()
+        st.cache_resource.clear()
+
+        # Check if the subfolder exists
+        if os.path.exists("Corporate_Documents"):
+            for filename in os.listdir("Corporate_Documents"):
+                file_path = os.path.join("Corporate_Documents", filename)
+                try:
+                    if os.path.isfile(file_path):
+                        os.unlink(file_path)
+                except Exception as e:
+                    st.error(f"Error deleting {file_path}: {e}")
+        else:
+            pass
+
+        if os.path.exists("Financial_Documents"):
+            # Iterate through files in the subfolder and delete them
+            for filename in os.listdir("Financial_Documents"):
+                file_path = os.path.join("Financial_Documents", filename)
+                try:
+                    if os.path.isfile(file_path):
+                        os.unlink(file_path)
+                except Exception as e:
+                    st.error(f"Error deleting {file_path}: {e}")
+        else:
+            pass
+            # st.warning("No 'data' subfolder found.")
+        
+        if os.path.exists("Intercreditor_Documents"):
+            # Iterate through files in the subfolder and delete them
+            for filename in os.listdir("Intercreditor_Documents"):
+                file_path = os.path.join("Intercreditor_Documents", filename)
+                try:
+                    if os.path.isfile(file_path):
+                        os.unlink(file_path)
+                except Exception as e:
+                    st.error(f"Error deleting {file_path}: {e}")
+        else:
+            pass
+            # st.warning("No 'data' subfolder found.")
+
+        folders_to_clean = ["data", "chroma_db_portfolio", "chroma_db_LT", "chroma_db_fin"]
+
+        for folder_path in folders_to_clean:
+            if os.path.exists(folder_path):
+                for item in os.listdir(folder_path):
+                    item_path = os.path.join(folder_path, item)
+                    try:
+                        if os.path.isfile(item_path) or os.path.islink(item_path):
+                            os.unlink(item_path)  # Remove files or symbolic links
+                        elif os.path.isdir(item_path):
+                            shutil.rmtree(item_path)  # Remove subfolders and all their contents
+                    except Exception as e:
+                        st.error(f"Error deleting {item_path}: {e}")
+            else:
+                pass
+                # st.warning(f"No '{folder_path}' folder found.")
+
+    with mrow1_col1:
+        st.subheader("Legal Document Analysis")
+        st.info("This tool is designed to provide a legal analysis of the documentation for  institutional investors.")
+        
+        button_container2 = st.container()
+        if "button_states" not in st.session_state:
+            st.session_state.button_states = {
+            "legal_analysis_button_key": False,
+            }
+        
+        if "results" not in st.session_state:
+            st.session_state.results = {}
+
+        if button_container2.button("Legal Analysis", key=legal_analysis_button_key):
+            st.session_state.button_states[legal_analysis_button_key] = True
+            result_generator = generate_strategy()  # Call the generator function
+            st.session_state.results["legal_analysis_output"] = result_generator
+            
+        if "legal_analysis_output" in st.session_state.results:           
+            st.write(st.session_state.results["legal_analysis_output"])
+        st.divider()
+        
+    with mrow1_col2:
+        if "legal_analysis_button_key" in st.session_state.results and st.session_state.results["legal_analysis_button_key"]:
+            
+            run_id = str(uuid.uuid4())
+
+            scratchpad = {
+                "questions": [],  # list of type Question
+                "answerpad": [],
+            }
+
+            if company_document:
+                store = create_vector_database_company()
+            if financial_document:
+                store = create_vector_database_financial()
+            if intercreditor_document:
+                store = create_vector_database_intercreditor()
+                
+            agent_settings = {
+                "max_iterations": 3,
+                "num_atomistic_questions": 2,
+                "num_questions_per_iteration": 4,
+                "question_atomizer_temperature": 0,
+                "question_creation_temperature": 0.4,
+                "question_prioritisation_temperature": 0,
+                "refine_answer_temperature": 0,
+                "qa_temperature": 0,
+                "analyser_temperature": 0,
+                "intermediate_answers_length": 200,
+                "answer_length": 500,
+            }
+
+            # Updated prompt templates to include chat history
+            def format_chat_history(chat_history):
+                """Format chat history as a single string for input to the chain."""
+                formatted_history = "\n".join([f"User: {entry['input']}\nAI: {entry['output']}" for entry in chat_history])
+                return formatted_history
+
+            # Initialize the agent with LCEL tools and memory
+            memory = ConversationBufferMemory(memory_key="chat_history", k=3, return_messages=True)
+            agent = Agent(agent_settings, scratchpad, store, True)
+            def conversational_chat(query):
+                # Get the result from the agent
+                agent.run({"input": query, "chat_history": st.session_state['history']})
+                
+                result = agent.get_latest_answer()
+
+                # Handle different response types
+                if isinstance(result, dict):
+                    # Extract the main content if the result is a dictionary
+                    result = result.get("output", "")  # Adjust the key as needed based on your agent's output
+                elif isinstance(result, list):
+                    # If the result is a list, join it into a single string
+                    result = "\n".join(result)
+                elif not isinstance(result, str):
+                    # Convert the result to a string if it is not already one
+                    result = str(result)
+                
+                # Add the query and the result to the session state
+                st.session_state['history'].append((query, result))
+                
+                # Update memory with the conversation
+                memory.save_context({"input": query}, {"output": result})
+                
+                # Return the result
+                return result
+
+            # Ensure session states are initialized
+            if 'history' not in st.session_state:
+                st.session_state['history'] = []
+
+            if 'generated' not in st.session_state:
+                st.session_state['generated'] = ["Let's discuss the legal and financial matters 🤗"]
+
+            if 'past' not in st.session_state:
+                st.session_state['past'] = ["Hey ! 👋"]
+
+            if 'input' not in st.session_state:
+                st.session_state['input'] = ""
+
+            # Streamlit layout
+            st.subheader("Discuss the documentation")
+            st.info("This tool is designed to enable discussion about the company's corporate and financial documentation.")
+            response_container = st.container()
+            container = st.container()
+
+            with container:
+                with st.form(key='my_form'):
+                    user_input = st.text_input("Query:", placeholder="What would you like to know about the documentation", key='input')
+                    submit_button = st.form_submit_button(label='Send')
+                if submit_button and user_input:
+                    output = conversational_chat(user_input)
+                    st.session_state['past'].append(user_input)
+                    st.session_state['generated'].append(output)
+                    user_input = "Query:"
+                #st.session_state['input'] = ""
+            # Display generated responses
+            if st.session_state['generated']:
+                with response_container:
+                    for i in range(len(st.session_state['generated'])):
+                        message(st.session_state["past"][i], is_user=True, key=str(i) + '_user', avatar_style="shapes")
+                        message(st.session_state["generated"][i], key=str(i), avatar_style="icons")

Assets/credentials.json

@@ -0,0 +1,6 @@
+{
+    "Mika": "password1",
+    "Maija": "MikaPassword",
+    "Sara": "password2",
+    "Teemu": "LT_334+DInv"
+}

chains_v2/create_questions.py

@@ -0,0 +1,40 @@
+from langchain.llms import BaseLLM
+from langchain.base_language import BaseLanguageModel
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+
+
+class QuestionCreationChain(LLMChain):
+    """Chain to generates subsequent questions."""
+    # Check what the below code line means and what it in practice does
+    @classmethod
+    def from_llm(cls, llm: BaseLLM, verbose: bool = True) -> LLMChain:
+        questions_creation_template = (
+            "You are a part of a team. The ultimate goal of your team is to"
+            " answer the following Question: '{question}'.\n"
+            "Your team has discovered some new text (delimited by ```) that may be relevant to your ultimate goal."
+            " text: \n ``` {context} ``` \n"
+            "Your task is to ask new questions that may help your team achieve the ultimate goal."
+            " If you think that the text is relevant to your ultimate goal, then ask new questions."
+            " New questions should be based only on the text and the goal Question and no other previous knowledge."
+            " The new questions should have no semantic overlap with questions in the following list:\n"
+            " {previous_questions}\n"
+            "You can ask up to {num_questions} new questions."
+            " Return the questions as a comma separated list. "
+            " Format your response as a numbered list of questions, like:\n"
+            "n. First question\n"
+            "n. Second question\n"
+            "Start the list with number {start_id}"
+        )
+
+        prompt = PromptTemplate(
+            template=questions_creation_template,
+            input_variables=[
+                "question",
+                "context",
+                "previous_questions",
+                "num_questions",
+                "start_id",
+            ],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

chains_v2/most_pertinent_question.py

@@ -0,0 +1,31 @@
+
+from langchain.llms import BaseLLM
+from langchain.base_language import BaseLanguageModel
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+
+class MostPertinentQuestion(LLMChain):
+    """
+    This chain picks one question out of a list of questions
+    most pertinent to the original question. 
+    """
+
+    @classmethod
+    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
+        """Get the response parser."""
+        question_prioritization_template = (
+            "You are provided with the following list of questions:"
+            " {unanswered_questions} \n"
+            " Your task is to choose one question from the above list" 
+            " that is the most pertinent to the following query:\n" 
+            " '{original_question}' \n"
+            " Respond with one question out of the provided list of questions."
+            " Return the questions as it is without any edits."
+            " Format your response like:\n"
+            " #. question"
+        )
+        prompt = PromptTemplate(
+            template=question_prioritization_template,
+            input_variables=["unanswered_questions", "original_question"],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

chains_v2/question_atomizer.py

@@ -0,0 +1,34 @@
+
+from langchain.llms import BaseLLM
+from langchain.base_language import BaseLanguageModel
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+
+
+class QuestionAtomizer(LLMChain):
+    """
+    This chain splits the original question into a set of atomistic questions. 
+    """
+
+    @classmethod
+    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
+        """Get the response parser."""
+        question_atomizer_template = (
+            " Your are provided with the following question:"
+            " '{question}' \n"
+            " Your task is to split the given question in at most {num_questions} very"
+            " simple, basic and atomist sub-questions (only if needed) using only the"
+            " information given in the question and no other prior knowledge."
+            " The sub-questions should be directly related to the intent of the original question."
+            " Consider the primary subject and the predicate of the question (if any) when creating sub questions.\n"
+            " Consider also the Characters, Ideas, Concepts, Entities, Actions, Or Events mentioned"
+            " in the question (if any) when creating the sub questions.\n"
+            " The sub questions should have no semantic overlap with each other."
+            " Format your response like: \n"
+            " n. question"
+        )
+        prompt = PromptTemplate(
+            template=question_atomizer_template,
+            input_variables=["question", "num_questions"],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

chains_v2/refine_answer.py

@@ -0,0 +1,30 @@
+from langchain.llms import BaseLLM
+from langchain.base_language import BaseLanguageModel
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+
+
+class RefineAnswer(LLMChain):
+    """
+    This refines the answer with every iteration.
+    """
+
+    @classmethod
+    def from_llm(cls, llm: BaseLanguageModel, verbose: bool = True) -> LLMChain:
+        """Get the response parser."""
+        prompt_template = (
+            "Your task is to answer the following question.\n"
+            " Question: '{question}'\n"
+            " You are provided with an existing Answer: \n---\n{answer}\n---\n\n"
+            " You are also provided with some additional context that may be relevant to the question.\n"
+            " New Context: \n---\n{context}\n---\n\n"
+            " You have the opportunity to rewrite and improve upon the existing answer."
+            " Use only the information from the existing answer and the given context to write better answer."
+            " Use a descriptive style and a business casual language."
+            " If the context isn't useful, return the existing answer."
+        )
+        prompt = PromptTemplate(
+            template=prompt_template,
+            input_variables=["question", "answer", "context"],
+        )
+        return cls(prompt=prompt, llm=llm, verbose=verbose)

chains_v2/research_compiler.py

@@ -0,0 +1,41 @@
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+
+def research_compiler(llm, question: str, notes: str, answer_length: int, verbose: bool = True):
+    # prompt_template = (
+    # "You are a researcher. Your task is to answer the following question\n"
+    # " Question: '{question}' \n"
+    # " You are provided with some notes (delimited between '___')"
+    # " ___\n{notes}\n ___\n"
+    # " The notes include answers to several questions that may be relevant to the original question."
+    # " Use only the information from the notes that is most pertinent to the question."
+    # " Write the answer solely based on the give notes and no other provious knowledge."
+    # " Answer should be clear, crisp and detailed."
+    # " Write your answer in less than {answer_length} words."
+    # " Answer :")
+    prompt_template = (
+        "You are a research agent who answers complex questions with clear, crisp and detailed answers."
+        " You are provided with a question and some research notes prepared by your team."
+        " Question: {question} \n"
+        " Notes: {notes} \n"
+        " Your task is to answer the question entirely based on the given notes."
+        " The notes contain a list of intermediate-questions and answers that may be helpful to you in writing an answer."
+        " Use only the most relevant information from the notes while writing your answer."
+        " Do not use any prior knowledge while writing your answer, Do not make up the answer."
+        " If the notes are not relevant to the question, just return 'Context is insufficient to answer the question'."
+        " Remember your goal is to answer the question as objectively as possible."
+        " Write your answer succinctly in less than {answer_length} words."
+    )
+
+    PROMPT = PromptTemplate(
+        template=prompt_template, input_variables=["notes", "question", "answer_length"]
+    )
+
+    chain = LLMChain(
+        llm=llm,
+        prompt=PROMPT,
+        verbose=verbose,
+    )
+
+    result = chain({"question": question, "notes": notes, "answer_length": answer_length})
+    return result

chains_v2/retrieval_qa.py

@@ -0,0 +1,37 @@
+# from langchain.llms import BaseLLM
+# from langchain.base_language import BaseLanguageModel
+# from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+from langchain.vectorstores import PGVector
+from langchain.chains import RetrievalQA
+
+def retrieval_qa(llm, retriever: PGVector, question: str, answer_length: 250, verbose: bool = True):
+    """
+    This chain is used to answer the intermediate questions.
+    """
+    prompt_answer_length = f" Answer as succinctly as possible in less than {answer_length} words.\n"
+
+    prompt_template = \
+        "You are provided with a question and some helpful context to answer the question \n" \
+        " Question: {question}\n" \
+        " Context: {context}\n" \
+        "Your task is to answer the question based in the information given in the context" \
+        " Answer the question entirely based on the context and no other previous knowledge." \
+        " If the context provided is empty or irrelevant, just return 'Context not sufficient'"\
+        + prompt_answer_length
+
+    PROMPT = PromptTemplate(
+        template=prompt_template, input_variables=["context", "question"]
+    )
+
+    qa_chain = RetrievalQA.from_chain_type(
+        llm=llm, 
+        chain_type="stuff", 
+        retriever=retriever,
+        return_source_documents=True,
+        chain_type_kwargs={"prompt": PROMPT},
+        verbose = verbose,
+    )
+
+    result = qa_chain({"query": question})
+    return result['result'], result['source_documents']

helpers/questions_helper.py

@@ -0,0 +1,23 @@
+
+getAnsweredQuestions = lambda questions: [
+    q for q in questions if q["status"] == "answered"
+]
+getUnansweredQuestions = lambda questions: [
+    q for q in questions if q["status"] == "unanswered"
+]
+getSubQuestions = lambda questions: [q for q in questions if q["type"] == "subquestion"]
+getHopQuestions = lambda questions: [q for q in questions if q["type"] == "hop"]
+getLastQuestionId = lambda questions: max([q["id"] for q in questions])
+
+
+def markAnswered(questions, id: int):
+    for q in questions:
+        if q["id"] == id:
+            q["status"] = "answered"
+
+
+def getQuestionById(questions, id: int):
+    q = [q for q in questions if q["id"] == id]
+    if len(q) == 0:
+        return None
+    return q[0]

helpers/response_helpers.py

@@ -0,0 +1,45 @@
+'''def qStr2Dict(question: str) -> dict:
+    print('qStr2Dict :', question)
+    split = question.strip(" '\"").split(".", 1)
+    question_dict = {'id': int(split[0]), 'question': split[-1].strip()}
+    return question_dict
+
+
+def result2QuestionsList(question_response: str, type: str, status: str) -> list:
+    response_splits = question_response.split("\n")
+    qlist = []
+    for q in response_splits:
+        question = {**qStr2Dict(q), 'type': type, 'status': status, 'answer': None}
+        qlist = qlist + [question]
+    return qlist
+'''
+def qStr2Dict(question: str) -> dict:
+    try:
+        print('qStr2Dict:', question)
+        split = question.strip(" '\"").split(".", 1)
+        
+        # Check if id_part is 'Here are the sub-questions:', set it to integer 1
+        id_part = split[0].replace("#", "").strip()
+        if isinstance(id_part, str):
+            id_part = 1  # Convert any string to integer 1
+        
+        # Convert id_part to integer, if not empty
+        id_part = int(id_part) if id_part else None
+        
+        question_dict = {'id': id_part, 'question': split[-1].strip()}
+        return question_dict
+    except ValueError as e:
+        print("Error:", e)
+        return {'id': None, 'question': question.strip()}
+
+def result2QuestionsList(question_response: str, type: str, status: str) -> list:
+    response_splits = question_response.split("\n")
+    qlist = []
+    for q in response_splits:
+        # Skip empty lines
+        if q.strip() == '':
+            continue
+        question = qStr2Dict(q)
+        question.update({'type': type, 'status': status, 'answer': None})
+        qlist.append(question)
+    return qlist