Mjlehtim commited on
Commit
c24a61d
β€’
1 Parent(s): 8ae946f

persist_directories added

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ chroma_db_company/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
AM_Document_Analysis_v1.2.py ADDED
@@ -0,0 +1,1178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import shutil
3
+ import streamlit as st
4
+ from fpdf import FPDF
5
+ from chromadb import Client
6
+ from chromadb.config import Settings
7
+ import json
8
+ import chromadb
9
+ from langchain_community.utilities import SerpAPIWrapper
10
+ from llama_index.core import VectorStoreIndex
11
+ from langchain_core.output_parsers import StrOutputParser
12
+ from langchain_core.runnables import RunnablePassthrough
13
+ from langchain_groq import ChatGroq
14
+ from langchain.chains import LLMChain
15
+ from langchain.agents import AgentType, Tool, initialize_agent, AgentExecutor
16
+ from llama_parse import LlamaParse
17
+ from langchain_community.document_loaders import UnstructuredMarkdownLoader
18
+ from langchain_huggingface import HuggingFaceEmbeddings
19
+ from llama_index.core import SimpleDirectoryReader
20
+ from dotenv import load_dotenv, find_dotenv
21
+ from streamlit_chat import message
22
+ from langchain_community.vectorstores import Chroma
23
+ from langchain_community.utilities import SerpAPIWrapper
24
+ from langchain.chains import RetrievalQA
25
+ from langchain_community.document_loaders import DirectoryLoader
26
+ from langchain_community.document_loaders import PyMuPDFLoader
27
+ from langchain_community.document_loaders import UnstructuredXMLLoader
28
+ from langchain_community.document_loaders import CSVLoader
29
+ from langchain.prompts import PromptTemplate
30
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
31
+ from langchain.memory import ConversationBufferMemory
32
+ from langchain.prompts import PromptTemplate
33
+ import joblib
34
+ import nltk
35
+ from dotenv import load_dotenv, find_dotenv
36
+ import uuid
37
+ from langchain_community.embeddings import HuggingFaceEmbeddings
38
+ from langchain_community.vectorstores import Chroma
39
+ from langchain.chat_models import ChatOpenAI
40
+ from langchain.embeddings import OpenAIEmbeddings
41
+ from langchain.prompts import PromptTemplate
42
+ from yachalk import chalk
43
+ from langchain.vectorstores import PGVector
44
+ from langchain.document_loaders import PyPDFLoader, UnstructuredPDFLoader, PyPDFium2Loader
45
+ from langchain_community.document_loaders import PyPDFDirectoryLoader
46
+ ## Import all the chains.
47
+ from chains_v2.create_questions import QuestionCreationChain
48
+ from chains_v2.most_pertinent_question import MostPertinentQuestion
49
+ from chains_v2.retrieval_qa import retrieval_qa
50
+ from chains_v2.research_compiler import research_compiler
51
+ from chains_v2.question_atomizer import QuestionAtomizer
52
+ from chains_v2.refine_answer import RefineAnswer
53
+ ## Import all the helpers.
54
+ from helpers.response_helpers import result2QuestionsList
55
+ from helpers.response_helpers import qStr2Dict
56
+ from helpers.questions_helper import getAnsweredQuestions
57
+ from helpers.questions_helper import getUnansweredQuestions
58
+ from helpers.questions_helper import getSubQuestions
59
+ from helpers.questions_helper import getHopQuestions
60
+ from helpers.questions_helper import getLastQuestionId
61
+ from helpers.questions_helper import markAnswered
62
+ from helpers.questions_helper import getQuestionById
63
+
64
+ import nest_asyncio # noqa: E402
65
+ nest_asyncio.apply()
66
+
67
+ load_dotenv()
68
+ load_dotenv(find_dotenv())
69
+
70
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
71
+ SERPAPI_API_KEY = os.environ["SERPAPI_API_KEY"]
72
+ GOOGLE_CSE_ID = os.environ["GOOGLE_CSE_ID"]
73
+ GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"]
74
+ LLAMA_PARSE_API_KEY = os.environ["LLAMA_PARSE_API_KEY"]
75
+ HUGGINGFACEHUB_API_TOKEN = os.environ["HUGGINGFACEHUB_API_TOKEN"]
76
+ LANGCHAIN_API_KEY = os.environ["LANGCHAIN_API_KEY"]
77
+ LANGCHAIN_ENDPOINT = os.environ["LANGCHAIN_ENDPOINT"]
78
+ LANGCHAIN_PROJECT = os.environ["LANGCHAIN_PROJECT"]
79
+ groq_api_key=os.getenv('GROQ_API_KEY')
80
+
81
+ st.set_page_config(layout="wide")
82
+
83
+ css = """
84
+ <style>
85
+ [data-testid="stAppViewContainer"] {
86
+ background-color: #f8f9fa; /* Very light grey */
87
+ }
88
+ [data-testid="stSidebar"] {
89
+ background-color: white;
90
+ color: black;
91
+ }
92
+ [data-testid="stAppViewContainer"] * {
93
+ color: black; /* Ensure all text is black */
94
+ }
95
+ button {
96
+ background-color: #add8e6; /* Light blue for primary buttons */
97
+ color: black;
98
+ border: 2px solid green; /* Green border */
99
+ }
100
+ button:hover {
101
+ background-color: #87ceeb; /* Slightly darker blue on hover */
102
+ }
103
+
104
+ button:active {
105
+ outline: 2px solid green; /* Green outline when the button is pressed */
106
+ outline-offset: 2px; /* Space between button and outline */
107
+ }
108
+
109
+ .stButton>button:first-child {
110
+ background-color: #add8e6; /* Light blue for primary buttons */
111
+ color: black;
112
+ }
113
+ .stButton>button:first-child:hover {
114
+ background-color: #87ceeb; /* Slightly darker blue on hover */
115
+ }
116
+ .stButton>button:nth-child(2) {
117
+ background-color: #b0e0e6; /* Even lighter blue for secondary buttons */
118
+ color: black;
119
+ }
120
+ .stButton>button:nth-child(2):hover {
121
+ background-color: #add8e6; /* Slightly darker blue on hover */
122
+ }
123
+ [data-testid="stFileUploadDropzone"] {
124
+ background-color: white; /* White background for file upload */
125
+ }
126
+ [data-testid="stFileUploadDropzone"] .stDropzone, [data-testid="stFileUploadDropzone"] .stDropzone input {
127
+ color: black; /* Ensure file upload text is black */
128
+ }
129
+
130
+ .stButton>button:active {
131
+ outline: 2px solid green; /* Green outline when the button is pressed */
132
+ outline-offset: 2px;
133
+ }
134
+ </style>
135
+ """
136
+ def load_credentials(filepath):
137
+ with open(filepath, 'r') as file:
138
+ return json.load(file)
139
+
140
+ # Load credentials from 'credentials.json'
141
+ credentials = load_credentials('Assets/credentials.json')
142
+
143
+ # Initialize session state if not already done
144
+ if 'logged_in' not in st.session_state:
145
+ st.session_state.logged_in = False
146
+ st.session_state.username = ''
147
+
148
+ # Function to handle login
149
+ def login(username, password):
150
+ if username in credentials and credentials[username] == password:
151
+ st.session_state.logged_in = True
152
+ st.session_state.username = username
153
+ st.rerun() # Rerun to reflect login state
154
+ else:
155
+ st.session_state.logged_in = False
156
+ st.session_state.username = ''
157
+ st.error("Invalid username or password.")
158
+
159
+ # Function to handle logout
160
+ def logout():
161
+ st.session_state.logged_in = False
162
+ st.session_state.username = ''
163
+ st.rerun() # Rerun to reflect logout state
164
+
165
+ #--------------
166
+ ## Define log printers
167
+
168
+ def print_iteration(current_iteration):
169
+ print(
170
+ chalk.bg_yellow_bright.black.bold(
171
+ f"\n Iteration - {current_iteration} β–·β–Ά \n"
172
+ )
173
+ )
174
+
175
+
176
+ def print_unanswered_questions(unanswered):
177
+ print(
178
+ chalk.cyan_bright("** Unanswered Questions **"),
179
+ chalk.cyan("".join([f"\n'{q['id']}. {q['question']}'" for q in unanswered])),
180
+ )
181
+
182
+
183
+ def print_next_question(current_question_id, current_question):
184
+ print(
185
+ chalk.magenta.bold("** πŸ€” Next Questions I must ask: **\n"),
186
+ chalk.magenta(current_question_id),
187
+ chalk.magenta(current_question["question"]),
188
+ )
189
+
190
+
191
+ def print_answer(current_question):
192
+ print(
193
+ chalk.yellow_bright.bold("** Answer **\n"),
194
+ chalk.yellow_bright(current_question["answer"]),
195
+ )
196
+
197
+
198
+ def print_final_answer(answerpad):
199
+ print(
200
+ chalk.white("** Refined Answer **\n"),
201
+ chalk.white(answerpad[-1]),
202
+ )
203
+
204
+
205
+ def print_max_iterations():
206
+ print(
207
+ chalk.bg_yellow_bright.black.bold(
208
+ "\n βœ”βœ” Max Iterations Reached. Compiling the results ...\n"
209
+ )
210
+ )
211
+
212
+
213
+ def print_result(result):
214
+ print(chalk.italic.white_bright((result["text"])))
215
+
216
+
217
+ def print_sub_question(q):
218
+ print(chalk.magenta.bold(f"** Sub Question **\n{q['question']}\n{q['answer']}\n"))
219
+ #--------------
220
+ ## ---- The researcher ----- ##
221
+
222
+ class Agent:
223
+ ## Create chains
224
+ def __init__(self, agent_settings, scratchpad, store, verbose):
225
+ self.store = store
226
+ self.scratchpad = scratchpad
227
+ self.agent_settings = agent_settings
228
+ self.verbose = verbose
229
+ self.question_creation_chain = QuestionCreationChain.from_llm(
230
+ language_model(
231
+ temperature=self.agent_settings["question_creation_temperature"]
232
+ ),
233
+ verbose=self.verbose,
234
+ )
235
+ self.question_atomizer = QuestionAtomizer.from_llm(
236
+ llm=language_model(
237
+ temperature=self.agent_settings["question_atomizer_temperature"]
238
+ ),
239
+ verbose=self.verbose,
240
+ )
241
+ self.most_pertinent_question = MostPertinentQuestion.from_llm(
242
+ language_model(
243
+ temperature=self.agent_settings["question_creation_temperature"]
244
+ ),
245
+ verbose=self.verbose,
246
+ )
247
+ self.refine_answer = RefineAnswer.from_llm(
248
+ language_model(
249
+ temperature=self.agent_settings["refine_answer_temperature"]
250
+ ),
251
+ verbose=self.verbose,
252
+ )
253
+
254
+ def run(self, question):
255
+ ## Step 0. Prepare the initial set of questions
256
+ atomized_questions_response = self.question_atomizer.run(
257
+ question=question,
258
+ num_questions=self.agent_settings["num_atomistic_questions"],
259
+ )
260
+
261
+ self.scratchpad["questions"] += result2QuestionsList(
262
+ question_response=atomized_questions_response,
263
+ type="subquestion",
264
+ status="unanswered",
265
+ )
266
+
267
+ for q in self.scratchpad["questions"]:
268
+ q["answer"], q["documents"] = retrieval_qa(
269
+ llm=language_model(
270
+ temperature=self.agent_settings["qa_temperature"],
271
+ verbose=self.verbose,
272
+ ),
273
+ retriever=self.store.as_retriever(
274
+ search_type="mmr", search_kwargs={"k": 5, "fetch_k": 10}
275
+ ),
276
+ question=q["question"],
277
+ answer_length=self.agent_settings["intermediate_answers_length"],
278
+ verbose=self.verbose,
279
+ )
280
+ q["status"] = "answered"
281
+ print_sub_question(q)
282
+
283
+
284
+ current_context = "".join(
285
+ f"\n{q['id']}. {q['question']}\n{q['answer']}\n"
286
+ for q in self.scratchpad["questions"]
287
+ )
288
+
289
+ self.scratchpad["answerpad"] += [current_context]
290
+
291
+ current_iteration = 0
292
+
293
+ while True:
294
+ current_iteration += 1
295
+ print_iteration(current_iteration)
296
+
297
+ # STEP 1: create questions
298
+ start_id = getLastQuestionId(self.scratchpad["questions"]) + 1
299
+ questions_response = self.question_creation_chain.run(
300
+ question=question,
301
+ context=current_context,
302
+ previous_questions=[
303
+ "".join(f"\n{q['question']}") for q in self.scratchpad["questions"]
304
+ ],
305
+ num_questions=self.agent_settings["num_questions_per_iteration"],
306
+ start_id=start_id,
307
+ )
308
+ self.scratchpad["questions"] += result2QuestionsList(
309
+ question_response=questions_response,
310
+ type="hop",
311
+ status="unanswered",
312
+ )
313
+
314
+ # STEP 2: Choose question for current iteration
315
+ unanswered = getUnansweredQuestions(self.scratchpad["questions"])
316
+ unanswered_questions_prompt = self.unanswered_questions_prompt(unanswered)
317
+ print_unanswered_questions(unanswered)
318
+ response = self.most_pertinent_question.run(
319
+ original_question=question,
320
+ unanswered_questions=unanswered_questions_prompt,
321
+ )
322
+ current_question_dict = qStr2Dict(question=response)
323
+ current_question_id = current_question_dict["id"]
324
+ current_question = getQuestionById(
325
+ self.scratchpad["questions"], current_question_id
326
+ )
327
+ print_next_question(current_question_id, current_question)
328
+
329
+ # STEP 3: Answer the question
330
+ current_question["answer"], current_question["documents"] = retrieval_qa(
331
+ llm=language_model(
332
+ temperature=self.agent_settings["qa_temperature"],
333
+ verbose=self.verbose,
334
+ ),
335
+ retriever=self.store.as_retriever(
336
+ search_type="mmr", search_kwargs={"k": 5, "fetch_k": 10}
337
+ ),
338
+ question=current_question["question"],
339
+ answer_length=self.agent_settings["intermediate_answers_length"],
340
+ verbose=self.verbose,
341
+ )
342
+ markAnswered(self.scratchpad["questions"], current_question_id)
343
+ print_answer(current_question)
344
+ current_context = current_question["answer"]
345
+
346
+ ## STEP 4: refine the answer
347
+ refinement_context = current_question["question"] + "\n" + current_context
348
+ refine_answer = self.refine_answer.run(
349
+ question=question,
350
+ context=refinement_context,
351
+ answer=self.get_latest_answer(),
352
+ )
353
+ self.scratchpad["answerpad"] += [refine_answer]
354
+ print_final_answer(self.scratchpad["answerpad"])
355
+
356
+ if current_iteration > self.agent_settings["max_iterations"]:
357
+ print_max_iterations()
358
+ break
359
+
360
+ def unanswered_questions_prompt(self, unanswered):
361
+ return (
362
+ "[" + "".join([f"\n{q['id']}. {q['question']}" for q in unanswered]) + "]"
363
+ )
364
+
365
+ def notes_prompt(self, answered_questions):
366
+ return "".join(
367
+ [
368
+ f"{{ Question: {q['question']}, Answer: {q['answer']} }}"
369
+ for q in answered_questions
370
+ ]
371
+ )
372
+
373
+ def get_latest_answer(self):
374
+ answers = self.scratchpad["answerpad"]
375
+ answer = answers[-1] if answers else ""
376
+ return answer
377
+
378
+ #--------------
379
+ # If not logged in, show login form
380
+ if not st.session_state.logged_in:
381
+ st.sidebar.write("Login")
382
+ username = st.sidebar.text_input('Username')
383
+ password = st.sidebar.text_input('Password', type='password')
384
+ if st.sidebar.button('Login'):
385
+ login(username, password)
386
+ # Stop the script here if the user is not logged in
387
+ st.stop()
388
+
389
+
390
+ # If logged in, show logout button and main content
391
+ st.sidebar.image('StratXcel.png', width=150)
392
+ if st.session_state.logged_in:
393
+ st.sidebar.write(f"Welcome, {st.session_state.username}!")
394
+ if st.sidebar.button('Logout'):
395
+ logout()
396
+
397
+ st.write(css, unsafe_allow_html=True)
398
+
399
+ company_document = st.sidebar.toggle("Company document", False)
400
+ financial_document = st.sidebar.toggle("Financial document", False)
401
+ intercreditor_document = st.sidebar.toggle("Intercreditor document", False)
402
+
403
+ #-------------
404
+ llm=ChatGroq(groq_api_key=groq_api_key,
405
+ model_name="Llama-3.1-70b-Versatile", temperature = 0.0, streaming=True)
406
+
407
+ Llama = "Llama-3.1-70b-Versatile"
408
+ def language_model(
409
+ model_name: str = Llama, temperature: float = 0, verbose: bool = False
410
+ ):
411
+ llm=ChatGroq(groq_api_key=groq_api_key, model_name=model_name, temperature=temperature, verbose=verbose)
412
+ return llm
413
+ #--------------
414
+ doc_retriever_company = None
415
+ doc_retriever_financials = None
416
+ doc_retriever_intercreditor = None
417
+
418
+ #--------------
419
+
420
+ #@st.cache_data
421
+ def load_or_parse_data_company():
422
+ data_file = "./data/parsed_data_company.pkl"
423
+
424
+ parsingInstructionUber10k = """The provided documents are company law documents of a company.
425
+ They contain detailed information about the rights and obligations of the company and its shareholders and contracting parties.
426
+ They also contain procedures for dispute resolution, voting, control priority and exit and sale situations.
427
+ You must never provide false legal or financial information. Use only the information included in the context documents.
428
+ Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""
429
+
430
+ parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
431
+ result_type="markdown",
432
+ parsing_instruction=parsingInstructionUber10k,
433
+ max_timeout=5000,
434
+ gpt4o_mode=True,
435
+ )
436
+
437
+ file_extractor = {".pdf": parser}
438
+ reader = SimpleDirectoryReader("./Corporate_Documents", file_extractor=file_extractor)
439
+ documents = reader.load_data()
440
+
441
+ print("Saving the parse results in .pkl format ..........")
442
+ joblib.dump(documents, data_file)
443
+
444
+ # Set the parsed data to the variable
445
+ parsed_data_company = documents
446
+
447
+ return parsed_data_company
448
+
449
+ #@st.cache_data
450
+ def load_or_parse_data_financial():
451
+ data_file = "./data/parsed_data_financial.pkl"
452
+
453
+ parsingInstructionUber10k = """The provided documents are financial law documents of a company.
454
+ They contain detailed information about the rights and obligations of the company and its creditors.
455
+ They also contain procedures for acceleration of debt, sale of security, enforcement, use of creditor control, priority and distribution of assets.
456
+ You must never provide false legal or financial information. Use only the information included in the context documents.
457
+ Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""
458
+
459
+ parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
460
+ result_type="markdown",
461
+ parsing_instruction=parsingInstructionUber10k,
462
+ max_timeout=5000,
463
+ gpt4o_mode=True,
464
+ )
465
+
466
+ file_extractor = {".pdf": parser}
467
+ reader = SimpleDirectoryReader("./Financial_Documents", file_extractor=file_extractor)
468
+ documents = reader.load_data()
469
+
470
+ print("Saving the parse results in .pkl format ..........")
471
+ joblib.dump(documents, data_file)
472
+
473
+ # Set the parsed data to the variable
474
+ parsed_data_financial = documents
475
+
476
+ return parsed_data_financial
477
+
478
+ #--------------
479
+
480
+ #@st.cache_data
481
+ def load_or_parse_data_intercreditor():
482
+ data_file = "./data/parsed_data_intercreditor.pkl"
483
+
484
+ parsingInstructionUber10k = """The provided documents are intercreditor agreements a company .
485
+ They contain detailed information about the rights and obligations of the company and its creditors and creditor groups.
486
+ They also contain procedures for acceleration of debt, sale of security, enforcement, use of creditor control, priority and distribution of assets.
487
+ You must never provide false legal or financial information. Use only the information included in the context documents.
488
+ Only refer to other sources if the context document refers to them or if necessary to provide additional understanding to company's own contracts."""
489
+
490
+ parser = LlamaParse(api_key=LLAMA_PARSE_API_KEY,
491
+ result_type="markdown",
492
+ parsing_instruction=parsingInstructionUber10k,
493
+ max_timeout=5000,
494
+ gpt4o_mode=True,
495
+ )
496
+
497
+ file_extractor = {".pdf": parser}
498
+ reader = SimpleDirectoryReader("./Intercreditor_Documents", file_extractor=file_extractor)
499
+ documents = reader.load_data()
500
+
501
+ print("Saving the parse results in .pkl format ..........")
502
+ joblib.dump(documents, data_file)
503
+
504
+ # Set the parsed data to the variable
505
+ parsed_data_financial = documents
506
+
507
+ return parsed_data_financial
508
+ #--------------
509
+ # Create vector database
510
+
511
+ @st.cache_resource
512
+ def create_vector_database_company():
513
+
514
+ llama_parse_documents = load_or_parse_data_company()
515
+
516
+ with open('data/output_company.md', 'a') as f: # Open the file in append mode ('a')
517
+ for doc in llama_parse_documents:
518
+ f.write(doc.text + '\n')
519
+
520
+ markdown_path = "data/output_company.md"
521
+ loader = UnstructuredMarkdownLoader(markdown_path)
522
+ documents = loader.load()
523
+
524
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=30)
525
+ docs = text_splitter.split_documents(documents)
526
+
527
+ print(f"length of documents loaded: {len(documents)}")
528
+ print(f"total number of document chunks generated :{len(docs)}")
529
+
530
+ persist_directory = "./chroma_db_company" # Specify directory for Chroma persistence
531
+ embed_model = HuggingFaceEmbeddings()
532
+ print('Vector DB not yet created !')
533
+
534
+ vs = Chroma.from_documents(
535
+ documents=docs,
536
+ embedding_function=embed_model,
537
+ collection_name="rag_company",
538
+ persist_directory=persist_directory # Ensure persistence
539
+ )
540
+
541
+ doc_retriever_company = vs
542
+ #doc_retriever_company = vs.as_retriever()
543
+
544
+ print('Vector DB created successfully !')
545
+ return doc_retriever_company
546
+
547
+ @st.cache_resource
548
+ def create_vector_database_financial():
549
+ # Call the function to either load or parse the data
550
+ llama_parse_documents = load_or_parse_data_financial()
551
+
552
+ with open('data/output_financials.md', 'a') as f: # Open the file in append mode ('a')
553
+ for doc in llama_parse_documents:
554
+ f.write(doc.text + '\n')
555
+
556
+ markdown_path = "data/output_financials.md"
557
+ loader = UnstructuredMarkdownLoader(markdown_path)
558
+ documents = loader.load()
559
+ # Split loaded documents into chunks
560
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15)
561
+ docs = text_splitter.split_documents(documents)
562
+
563
+ print(f"length of documents loaded: {len(documents)}")
564
+ print(f"total number of document chunks generated :{len(docs)}")
565
+ persist_directory = "./chroma_db_financial" # Specify directory for Chroma persistence
566
+
567
+ embed_model = HuggingFaceEmbeddings()
568
+
569
+ vs = Chroma.from_documents(
570
+ documents=docs,
571
+ embedding_function=embed_model,
572
+ collection_name="rag_financial",
573
+ persist_directory=persist_directory # Ensure persistence
574
+ )
575
+ doc_retriever_financial = vs
576
+ #doc_retriever_financial = vs.as_retriever()
577
+
578
+ print('Vector DB created successfully !')
579
+ return doc_retriever_financial
580
+
581
+ #--------------
582
+
583
+ @st.cache_resource
584
+ def create_vector_database_intercreditor():
585
+ # Call the function to either load or parse the data
586
+ llama_parse_documents = load_or_parse_data_intercreditor()
587
+
588
+ with open('data/output_intercreditor.md', 'a') as f: # Open the file in append mode ('a')
589
+ for doc in llama_parse_documents:
590
+ f.write(doc.text + '\n')
591
+
592
+ markdown_path = "data/output_intercreditor.md"
593
+ loader = UnstructuredMarkdownLoader(markdown_path)
594
+ documents = loader.load()
595
+ # Split loaded documents into chunks
596
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15)
597
+ docs = text_splitter.split_documents(documents)
598
+
599
+ print(f"length of documents loaded: {len(documents)}")
600
+ print(f"total number of document chunks generated :{len(docs)}")
601
+ persist_directory = "./chroma_db_intercreditor" # Specify directory for Chroma persistence
602
+ embed_model = HuggingFaceEmbeddings()
603
+
604
+ vs = Chroma.from_documents(
605
+ documents=docs,
606
+ embedding_function=embed_model,
607
+ collection_name="rag_intercreditor",
608
+ persist_directory=persist_directory # Ensure persistence
609
+ )
610
+ doc_retriever_intercreditor = vs
611
+ #doc_retriever_intercreditor = vs.as_retriever()
612
+
613
+ print('Vector DB created successfully !')
614
+ return doc_retriever_intercreditor
615
+
616
+ #--------------
617
+ legal_analysis_button_key = "legal_strategy_button"
618
+
619
+ #---------------
620
+ def delete_files_and_folders(folder_path):
621
+ for root, dirs, files in os.walk(folder_path, topdown=False):
622
+ for file in files:
623
+ try:
624
+ os.unlink(os.path.join(root, file))
625
+ except Exception as e:
626
+ st.error(f"Error deleting {os.path.join(root, file)}: {e}")
627
+ for dir in dirs:
628
+ try:
629
+ os.rmdir(os.path.join(root, dir))
630
+ except Exception as e:
631
+ st.error(f"Error deleting directory {os.path.join(root, dir)}: {e}")
632
+ #---------------
633
+
634
+ if company_document:
635
+ uploaded_files_ESG = st.sidebar.file_uploader("Choose company law documents", accept_multiple_files=True, key="company_files")
636
+ for uploaded_file in uploaded_files_ESG:
637
+ st.write("filename:", uploaded_file.name)
638
+ def save_uploadedfile(uploadedfile):
639
+ with open(os.path.join("Corporate_Documents",uploadedfile.name),"wb") as f:
640
+ f.write(uploadedfile.getbuffer())
641
+ return st.success("Saved File:{} to Company_Documents".format(uploadedfile.name))
642
+ save_uploadedfile(uploaded_file)
643
+
644
+ if financial_document:
645
+ uploaded_files_financials = st.sidebar.file_uploader("Choose financial law documents", accept_multiple_files=True, key="financial_files")
646
+ for uploaded_file in uploaded_files_financials:
647
+ st.write("filename:", uploaded_file.name)
648
+ def save_uploadedfile(uploadedfile):
649
+ with open(os.path.join("Financial_Documents",uploadedfile.name),"wb") as f:
650
+ f.write(uploadedfile.getbuffer())
651
+ return st.success("Saved File:{} to Financial_Documents".format(uploadedfile.name))
652
+ save_uploadedfile(uploaded_file)
653
+
654
+ if intercreditor_document:
655
+ uploaded_files_intercreditor = st.sidebar.file_uploader("Choose intercreditor documents", accept_multiple_files=True, key="intercreditor_files")
656
+ for uploaded_file in uploaded_files_intercreditor:
657
+ st.write("filename:", uploaded_file.name)
658
+ def save_uploadedfile(uploadedfile):
659
+ with open(os.path.join("Intercreditor_Documents",uploadedfile.name),"wb") as f:
660
+ f.write(uploadedfile.getbuffer())
661
+ return st.success("Saved File:{} to Intercreditor_Documents".format(uploadedfile.name))
662
+ save_uploadedfile(uploaded_file)
663
+ #---------------
664
+ def company_strategy():
665
+ doc_retriever_company = create_vector_database_company().as_retriever()
666
+ prompt_template = """<|system|>
667
+ You are a seasoned attorney specilizing in company and corporate law and legal analysis. You write expert analyses for institutional investors.
668
+ Output must have sub-headings in bold font and be fluent.<|end|>
669
+ <|user|>
670
+ Answer the {question} based on the information you find in context: {context} <|end|>
671
+ <|assistant|>"""
672
+
673
+ prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])
674
+
675
+ qa = (
676
+ {
677
+ "context": doc_retriever_company,
678
+ "question": RunnablePassthrough(),
679
+ }
680
+ | prompt
681
+ | llm
682
+ | StrOutputParser()
683
+ )
684
+
685
+ Corporate_answer_1 = qa.invoke("What provisions govern the appointment and removal of directors in the company? Outline the procedures and any required shareholder involvement in these processes.")
686
+
687
+ Corporate_answer_2 = qa.invoke("Explain the company's share capital structure, including any provisions for different classes of shares and the rights attached to them. How are voting rights distributed among shareholders?")
688
+
689
+ Corporate_answer_3 = qa.invoke("What restrictions or conditions are placed on the transfer or sale of shares in the company's articles of association or shareholders' agreements? Include any pre-emption rights or lock-in provisions.")
690
+
691
+ Corporate_answer_4 = qa.invoke("Describe the rights and obligations of majority and minority shareholders as outlined in the company's shareholders' agreements. What protections are in place for minority shareholders?")
692
+
693
+ Corporate_answer_5 = qa.invoke("What are the provisions for issuing new shares or increasing the company's capital? Detail any existing shareholder approval requirements or pre-emptive rights outlined in the company's governing documents.")
694
+
695
+ Corporate_answer_6 = qa.invoke("Outline the procedures for decision-making in shareholder meetings, including quorum requirements and voting thresholds for ordinary and special resolutions. How are dissenting shareholders addressed in key decisions?")
696
+
697
+ Corporate_answer_7 = qa.invoke("What mechanisms are in place for resolving shareholder disputes? Provide details on any arbitration or mediation clauses found in the company's articles or shareholders' agreements.")
698
+
699
+ Corporate_answer_8 = qa.invoke("Describe the exit mechanisms available for shareholders, such as drag-along and tag-along rights, and the circumstances under which they can be triggered.")
700
+
701
+ Corporate_answer_9 = qa.invoke("What rights do shareholders have to appoint or remove members of the board? Outline any requirements for shareholder approval in relation to board appointments or dismissals.")
702
+
703
+ Corporate_answer_10 = qa.invoke("Explain any restrictions on the powers of the board as set out in the company's governing documents. Are there specific decisions that require shareholder approval or consultation?")
704
+
705
+ Corporate_answer_11 = qa.invoke("What provisions are in place regarding dividends and the distribution of profits? How are dividend rights structured among different classes of shares, if applicable?")
706
+
707
+
708
+ corporate_output = f"""**__Director Appointment and Removal:__** {Corporate_answer_1} \n\n
709
+ **__Share Capital Structure and Voting Rights:__** {Corporate_answer_2} \n\n
710
+ **__Restrictions on Share Transfer and Sale:__** {Corporate_answer_3} \n\n
711
+ **__Rights and Obligations of Shareholders:__** {Corporate_answer_4} \n\n
712
+ **__Issuing New Shares and Capital Increases:__** {Corporate_answer_5} \n\n
713
+ **__Decision-Making in Shareholder Meetings:__** {Corporate_answer_6} \n\n
714
+ **__Shareholder Dispute Resolution Mechanisms:__** {Corporate_answer_7} \n\n
715
+ **__Exit Mechanisms for Shareholders:__** {Corporate_answer_8} \n\n
716
+ **__Rights to Appoint or Remove Board Members:__** {Corporate_answer_9} \n\n
717
+ **__Restrictions on the Board's Powers:__** {Corporate_answer_10} \n\n
718
+ **__Dividend and Profit Distribution Provisions:__** {Corporate_answer_11}"""
719
+
720
+ financial_output = corporate_output
721
+
722
+ with open("company_analysis.txt", 'w') as file:
723
+ file.write(financial_output)
724
+
725
+ return financial_output
726
+
727
+ def financial_strategy():
728
+ doc_retriever_financial = create_vector_database_financial().as_retriever()
729
+ prompt_template = """<|system|>
730
+ You are a seasoned attorney specilizing in financial law and legal analysis. You write expert analyses for institutional investors.
731
+ Output must have sub-headings in bold font and be fluent.<|end|>
732
+ <|user|>
733
+ Answer the {question} based on the information you find in context: {context} <|end|>
734
+ <|assistant|>"""
735
+
736
+ prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])
737
+
738
+ qa = (
739
+ {
740
+ "context": doc_retriever_financial,
741
+ "question": RunnablePassthrough(),
742
+ }
743
+ | prompt
744
+ | llm
745
+ | StrOutputParser()
746
+ )
747
+
748
+ Financial_answer_1 = qa.invoke("What are the parties of the agreements and key obligations of the borrower under the company's loan agreements? Describe any covenants or financial ratios the borrower must comply with.")
749
+
750
+ Financial_answer_3 = qa.invoke("What provisions govern the occurrence of events of default under the company's loan and bond agreements? Include any cross-default or material adverse change clauses.")
751
+
752
+ Financial_answer_4 = qa.invoke("Describe the rights of secured creditors under the company's security documents. What types of assets are secured, and how can creditors enforce their security in case of default?")
753
+
754
+ Financial_answer_5 = qa.invoke("What are the acceleration clauses in the company's financial agreements? Under what conditions can creditors demand early repayment or terminate financing arrangements?")
755
+
756
+ Financial_answer_6 = qa.invoke("Outline the procedures and requirements for enforcing security interests under the company's security documents. How do the rights of secured and unsecured creditors differ in this context?")
757
+
758
+ Financial_answer_7 = qa.invoke("How are decisions related to enforcement or restructuring prioritized among creditors?")
759
+
760
+ Financial_answer_8 = qa.invoke("Explain the company's obligations under any guarantees or indemnities provided to creditors. What are the limitations, if any, on the enforcement of these guarantees?")
761
+
762
+ Financial_answer_9 = qa.invoke("Describe the rights of bondholders or lenders in the company's bond issuance agreements or loans. What are the procedures for creditor meetings, and how can creditors exercise their rights in the event of default?")
763
+
764
+ Financial_answer_10 = qa.invoke("What protections are in place for junior creditors or subordinated debt holders in the company's financial agreements? How are their rights affected in the event of enforcement or restructuring?")
765
+
766
+ Financial_answer_11 = qa.invoke("What are the company's obligations to provide financial information to creditors under its loan or bond agreements? How frequently must the company report, and what information is typically required?")
767
+
768
+
769
+ financial_output = f"""**__Borrower Obligations and Covenants:__** {Financial_answer_1} \n\n
770
+ **__Events of Default and Cross-Default Provisions:__** {Financial_answer_3} \n\n
771
+ **__Rights of Secured Creditors and Enforcement of Security:__** {Financial_answer_4} \n\n
772
+ **__Acceleration Clauses and Early Repayment Triggers:__** {Financial_answer_5} \n\n
773
+ **__Enforcement of Security Interests:__** {Financial_answer_6} \n\n
774
+ **__Intercreditor Decision-Making and Control:__** {Financial_answer_7} \n\n
775
+ **__Guarantees and Indemnities Obligations:__** {Financial_answer_8} \n\n
776
+ **__Rights of Bondholders and Default Procedures:__** {Financial_answer_9} \n\n
777
+ **__Protections for Junior Creditors:__** {Financial_answer_10} \n\n
778
+ **__Financial Reporting Obligations to Creditors:__** {Financial_answer_11}"""
779
+
780
+ with open("financial_analysis.txt", 'w') as file:
781
+ file.write(financial_output)
782
+
783
+ return financial_output
784
+
785
+ def intercreditor_strategy():
786
+ doc_retriever_intercreditor = create_vector_database_intercreditor().as_retriever()
787
+ prompt_template = """<|system|>
788
+ You are a seasoned attorney specilizing in financial law and legal analysis. You write expert analyses for institutional investors.
789
+ Output must have sub-headings in bold font and be fluent.<|end|>
790
+ <|user|>
791
+ Answer the {question} based on the information you find in context: {context} <|end|>
792
+ <|assistant|>"""
793
+
794
+ prompt = PromptTemplate(template=prompt_template, input_variables=["question", "context"])
795
+
796
+ qa = (
797
+ {
798
+ "context": doc_retriever_intercreditor,
799
+ "question": RunnablePassthrough(),
800
+ }
801
+ | prompt
802
+ | llm
803
+ | StrOutputParser()
804
+ )
805
+
806
+ Intercreditor_answer_1 = qa.invoke("What are the parties involved in the intercreditor agreements, and what are their key roles and obligations?")
807
+
808
+ Intercreditor_answer_2 = qa.invoke("What provisions govern the ranking and priority of claims among creditors under the intercreditor agreements? Describe any subordination terms or waterfall clauses.")
809
+
810
+ Intercreditor_answer_3 = qa.invoke("How are enforcement actions handled under the intercreditor agreements? Are there specific procedures for appointing an enforcement agent or for coordinating enforcement between creditors?")
811
+
812
+ Intercreditor_answer_4 = qa.invoke("What are the standstill and turnover provisions in the intercreditor agreements? Under what circumstances can a creditor be prevented from taking enforcement actions?")
813
+
814
+ Intercreditor_answer_5 = qa.invoke("How are payment blockages or restrictions handled between senior and junior creditors? What are the limitations or conditions imposed on junior creditors under these provisions?")
815
+
816
+ Intercreditor_answer_6 = qa.invoke("What are the mechanisms for resolving disputes or conflicts between creditors in the intercreditor agreements?")
817
+
818
+ Intercreditor_answer_7 = qa.invoke("How do the intercreditor agreements handle the distribution of proceeds in the event of enforcement or restructuring? What are the priority rules for distributing recoveries among creditors?")
819
+
820
+ Intercreditor_answer_8 = qa.invoke("Describe the provisions related to amendments and waivers in the intercreditor agreements. How are decisions to amend the agreement or waive certain rights made among creditors?")
821
+
822
+ Intercreditor_answer_9 = qa.invoke("What restrictions or limitations exist in the intercreditor agreements concerning the ability of junior creditors to exercise rights in insolvency or restructuring proceedings?")
823
+
824
+ Intercreditor_answer_10 = qa.invoke("What reporting or information-sharing obligations exist under the intercreditor agreements? How frequently must creditors be updated, and what type of information is shared between creditor groups?")
825
+
826
+ intercreditor_output = f"""**__Parties and Obligations under Intercreditor Agreements:__** {Intercreditor_answer_1} \n\n
827
+ **__Ranking and Priority of Claims:__** {Intercreditor_answer_2} \n\n
828
+ **__Enforcement Actions and Coordination:__** {Intercreditor_answer_3} \n\n
829
+ **__Standstill and Turnover Provisions:__** {Intercreditor_answer_4} \n\n
830
+ **__Payment Blockages and Restrictions:__** {Intercreditor_answer_5} \n\n
831
+ **__Dispute Resolution Mechanisms:__** {Intercreditor_answer_6} \n\n
832
+ **__Distribution of Proceeds and Priority Rules:__** {Intercreditor_answer_7} \n\n
833
+ **__Amendments and Waivers:__** {Intercreditor_answer_8} \n\n
834
+ **__Restrictions on Junior Creditors in Insolvency:__** {Intercreditor_answer_9} \n\n
835
+ **__Information-Sharing Obligations:__** {Intercreditor_answer_10}"""
836
+
837
+ with open("intercreditor_analysis.txt", 'w') as file:
838
+ file.write(intercreditor_output)
839
+
840
+ return intercreditor_output
841
+
842
+ #-------------
843
+ @st.cache_data
844
+ def generate_strategy() -> str:
845
+ combined_output = ""
846
+
847
+ # Check if there are files in the Corporate_Documents folder
848
+ if company_document and os.path.exists("Corporate_Documents") and os.listdir("Corporate_Documents"):
849
+ company_output = company_strategy()
850
+ combined_output += company_output + '\n\n'
851
+
852
+ # Check if there are files in the Financial_Documents folder
853
+ if financial_document and os.path.exists("Financial_Documents") and os.listdir("Financial_Documents"):
854
+ financial_output = financial_strategy()
855
+ combined_output += financial_output + '\n\n'
856
+
857
+ # Check if there are files in the Intercreditor_Documents folder
858
+ if intercreditor_document and os.path.exists("Intercreditor_Documents") and os.listdir("Intercreditor_Documents"):
859
+ intercreditor_output = intercreditor_strategy()
860
+ combined_output += intercreditor_output + '\n\n'
861
+
862
+ # Set the combined result in a single session state key
863
+ st.session_state.results["legal_analysis_button_key"] = combined_output
864
+
865
+ return combined_output
866
+ #---------------
867
+ #@st.cache_data
868
+ def create_pdf():
869
+ company_content = ""
870
+ financial_content = ""
871
+ intercreditor_content = ""
872
+
873
+ # Check if 'company_analysis.txt' exists and open if it does
874
+ if os.path.exists('company_analysis.txt'):
875
+ with open('company_analysis.txt', 'r') as file1:
876
+ company_content = file1.read()
877
+
878
+ # Check if 'financial_analysis.txt' exists and open if it does
879
+ if os.path.exists('financial_analysis.txt'):
880
+ with open('financial_analysis.txt', 'r') as file2:
881
+ financial_content = file2.read()
882
+
883
+ # Check if 'intercreditor_analysis.txt' exists and open if it does
884
+ if os.path.exists('intercreditor_analysis.txt'):
885
+ with open('intercreditor_analysis.txt', 'r') as file3:
886
+ intercreditor_content = file3.read()
887
+
888
+ # Combine the contents of the available files
889
+ combined_content = ""
890
+
891
+ if company_content:
892
+ combined_content += company_content + '\n\n'
893
+
894
+ if financial_content:
895
+ combined_content += financial_content + '\n\n'
896
+
897
+ if intercreditor_content:
898
+ combined_content += intercreditor_content + '\n\n'
899
+
900
+ # Write the combined content to a new file only if any content exists
901
+ if combined_content:
902
+ with open('legal_analysis.txt', 'w') as outfile:
903
+ outfile.write(combined_content.strip())
904
+
905
+ text_file = "legal_analysis.txt"
906
+ pdf = FPDF('P', 'mm', 'A4')
907
+ pdf.add_page()
908
+ pdf.set_margins(10, 10, 10)
909
+ pdf.set_font("Arial", size=15)
910
+
911
+ pdf.cell(0, 10, txt="Structured Legal Analysis", ln=2, align='C')
912
+ pdf.ln(5)
913
+
914
+ pdf.set_font("Arial", size=11)
915
+ try:
916
+ with open(text_file, 'r', encoding='utf-8') as f:
917
+ for line in f:
918
+ pdf.multi_cell(0, 6, txt=line.encode('latin-1', 'replace').decode('latin-1'), align='L')
919
+ pdf.ln(5)
920
+ except UnicodeEncodeError:
921
+ print("UnicodeEncodeError: Some characters could not be encoded in Latin-1. Skipping...")
922
+ pass # Skip the lines causing UnicodeEncodeError
923
+
924
+ output_pdf_path = "ESG_analysis.pdf"
925
+ pdf.output(output_pdf_path)
926
+
927
+ #----------------
928
+ #llm = build_llm()
929
+
930
+ if 'results' not in st.session_state:
931
+ st.session_state.results = {
932
+ "legal_analysis_button_key": {}
933
+ }
934
+
935
+ loaders = {'.pdf': PyMuPDFLoader,
936
+ '.xml': UnstructuredXMLLoader,
937
+ '.csv': CSVLoader,
938
+ }
939
+
940
+ def create_directory_loader(file_type, directory_path):
941
+ return DirectoryLoader(
942
+ path=directory_path,
943
+ glob=f"**/*{file_type}",
944
+ loader_cls=loaders[file_type],
945
+ )
946
+
947
+
948
+ strategies_container = st.container()
949
+ with strategies_container:
950
+ mrow1_col1, mrow1_col2 = st.columns(2)
951
+
952
+ st.sidebar.info("To get started, please upload the documents from the company you would like to analyze.")
953
+ button_container = st.sidebar.container()
954
+ if os.path.exists("company_analysis.txt") and os.path.exists("financial_analysis.txt"):
955
+ create_pdf()
956
+ with open("ESG_analysis.pdf", "rb") as pdf_file:
957
+ PDFbyte = pdf_file.read()
958
+
959
+ st.sidebar.download_button(label="Download Analyses",
960
+ data=PDFbyte,
961
+ file_name="strategy_sheet.pdf",
962
+ mime='application/octet-stream',
963
+ )
964
+
965
+ if button_container.button("Clear All"):
966
+
967
+ st.session_state.button_states = {
968
+ "legal_analysis_button_key": False,
969
+ }
970
+ st.session_state.button_states = {
971
+ "financial_analysis_button_key": False,
972
+ }
973
+ st.session_state.results = {}
974
+
975
+ st.session_state['history'] = []
976
+ st.session_state['generated'] = ["Let's discuss the company documents πŸ€—"]
977
+ st.session_state['past'] = ["Hey ! πŸ‘‹"]
978
+ st.cache_data.clear()
979
+ st.cache_resource.clear()
980
+
981
+ # Check if the subfolder exists
982
+ if os.path.exists("Corporate_Documents"):
983
+ for filename in os.listdir("Corporate_Documents"):
984
+ file_path = os.path.join("Corporate_Documents", filename)
985
+ try:
986
+ if os.path.isfile(file_path):
987
+ os.unlink(file_path)
988
+ except Exception as e:
989
+ st.error(f"Error deleting {file_path}: {e}")
990
+ else:
991
+ pass
992
+
993
+ if os.path.exists("Financial_Documents"):
994
+ # Iterate through files in the subfolder and delete them
995
+ for filename in os.listdir("Financial_Documents"):
996
+ file_path = os.path.join("Financial_Documents", filename)
997
+ try:
998
+ if os.path.isfile(file_path):
999
+ os.unlink(file_path)
1000
+ except Exception as e:
1001
+ st.error(f"Error deleting {file_path}: {e}")
1002
+ else:
1003
+ pass
1004
+ # st.warning("No 'data' subfolder found.")
1005
+
1006
+ if os.path.exists("Intercreditor_Documents"):
1007
+ # Iterate through files in the subfolder and delete them
1008
+ for filename in os.listdir("Intercreditor_Documents"):
1009
+ file_path = os.path.join("Intercreditor_Documents", filename)
1010
+ try:
1011
+ if os.path.isfile(file_path):
1012
+ os.unlink(file_path)
1013
+ except Exception as e:
1014
+ st.error(f"Error deleting {file_path}: {e}")
1015
+ else:
1016
+ pass
1017
+ # st.warning("No 'data' subfolder found.")
1018
+
1019
+ folders_to_clean = ["data", "chroma_db_company", "chroma_db_financial", "chroma_db_intercreditor"]
1020
+
1021
+ for folder_path in folders_to_clean:
1022
+ if os.path.exists(folder_path):
1023
+ for item in os.listdir(folder_path):
1024
+ item_path = os.path.join(folder_path, item)
1025
+ try:
1026
+ if os.path.isfile(item_path) or os.path.islink(item_path):
1027
+ os.unlink(item_path) # Remove files or symbolic links
1028
+ elif os.path.isdir(item_path):
1029
+ shutil.rmtree(item_path) # Remove subfolders and all their contents
1030
+ except Exception as e:
1031
+ st.error(f"Error deleting {item_path}: {e}")
1032
+ else:
1033
+ pass
1034
+ # st.warning(f"No '{folder_path}' folder found.")
1035
+
1036
+ with mrow1_col1:
1037
+ st.subheader("Legal Document Analysis")
1038
+ st.info("This tool is designed to provide a legal analysis of the documentation for institutional investors.")
1039
+
1040
+ button_container2 = st.container()
1041
+ if "button_states" not in st.session_state:
1042
+ st.session_state.button_states = {
1043
+ "legal_analysis_button_key": False,
1044
+ }
1045
+
1046
+ if "results" not in st.session_state:
1047
+ st.session_state.results = {}
1048
+
1049
+ if button_container2.button("Legal Analysis", key=legal_analysis_button_key):
1050
+ st.session_state.button_states[legal_analysis_button_key] = True
1051
+ result_generator = generate_strategy() # Call the generator function
1052
+ st.session_state.results["legal_analysis_output"] = result_generator
1053
+
1054
+ if "legal_analysis_output" in st.session_state.results:
1055
+ st.write(st.session_state.results["legal_analysis_output"])
1056
+ st.divider()
1057
+
1058
+ with mrow1_col2:
1059
+ if "legal_analysis_button_key" in st.session_state.results and st.session_state.results["legal_analysis_button_key"]:
1060
+
1061
+ run_id = str(uuid.uuid4())
1062
+
1063
+ scratchpad = {
1064
+ "questions": [], # list of type Question
1065
+ "answerpad": [],
1066
+ }
1067
+
1068
+ embed_model = HuggingFaceEmbeddings()
1069
+
1070
+ vs_company = Chroma(
1071
+ persist_directory="./chroma_db_company", # Directory for persistent storage
1072
+ collection_name="rag_company",
1073
+ embedding_function=embed_model
1074
+ )
1075
+ vs_financial = Chroma(
1076
+ persist_directory="./chroma_db_financial", # Directory for persistent storage
1077
+ collection_name="rag_financial",
1078
+ embedding_function=embed_model
1079
+ )
1080
+ vs_intercreditor = Chroma(
1081
+ persist_directory="./chroma_db_intercreditor", # Directory for persistent storage
1082
+ collection_name="rag_intercreditor",
1083
+ embedding_function=embed_model
1084
+ )
1085
+
1086
+ if company_document:
1087
+ store = vs_company
1088
+ elif financial_document:
1089
+ store = vs_financial
1090
+ elif intercreditor_document:
1091
+ store = vs_intercreditor
1092
+ else:
1093
+ store = None
1094
+
1095
+ agent_settings = {
1096
+ "max_iterations": 3,
1097
+ "num_atomistic_questions": 2,
1098
+ "num_questions_per_iteration": 4,
1099
+ "question_atomizer_temperature": 0,
1100
+ "question_creation_temperature": 0.4,
1101
+ "question_prioritisation_temperature": 0,
1102
+ "refine_answer_temperature": 0,
1103
+ "qa_temperature": 0,
1104
+ "analyser_temperature": 0,
1105
+ "intermediate_answers_length": 200,
1106
+ "answer_length": 500,
1107
+ }
1108
+
1109
+ # Updated prompt templates to include chat history
1110
+ def format_chat_history(chat_history):
1111
+ """Format chat history as a single string for input to the chain."""
1112
+ formatted_history = "\n".join([f"User: {entry['input']}\nAI: {entry['output']}" for entry in chat_history])
1113
+ return formatted_history
1114
+
1115
+ # Initialize the agent with LCEL tools and memory
1116
+ memory = ConversationBufferMemory(memory_key="chat_history", k=3, return_messages=True)
1117
+ agent = Agent(agent_settings, scratchpad, store, True)
1118
+ def conversational_chat(query):
1119
+ # Get the result from the agent
1120
+ agent.run({"input": query, "chat_history": st.session_state['history']})
1121
+
1122
+ result = agent.get_latest_answer()
1123
+
1124
+ # Handle different response types
1125
+ if isinstance(result, dict):
1126
+ # Extract the main content if the result is a dictionary
1127
+ result = result.get("output", "") # Adjust the key as needed based on your agent's output
1128
+ elif isinstance(result, list):
1129
+ # If the result is a list, join it into a single string
1130
+ result = "\n".join(result)
1131
+ elif not isinstance(result, str):
1132
+ # Convert the result to a string if it is not already one
1133
+ result = str(result)
1134
+
1135
+ # Add the query and the result to the session state
1136
+ st.session_state['history'].append((query, result))
1137
+
1138
+ # Update memory with the conversation
1139
+ memory.save_context({"input": query}, {"output": result})
1140
+
1141
+ # Return the result
1142
+ return result
1143
+
1144
+ # Ensure session states are initialized
1145
+ if 'history' not in st.session_state:
1146
+ st.session_state['history'] = []
1147
+
1148
+ if 'generated' not in st.session_state:
1149
+ st.session_state['generated'] = ["Let's discuss the legal and financial matters πŸ€—"]
1150
+
1151
+ if 'past' not in st.session_state:
1152
+ st.session_state['past'] = ["Hey ! πŸ‘‹"]
1153
+
1154
+ if 'input' not in st.session_state:
1155
+ st.session_state['input'] = ""
1156
+
1157
+ # Streamlit layout
1158
+ st.subheader("Discuss the documentation")
1159
+ st.info("This tool is designed to enable discussion about the company's corporate and financial documentation.")
1160
+ response_container = st.container()
1161
+ container = st.container()
1162
+
1163
+ with container:
1164
+ with st.form(key='my_form'):
1165
+ user_input = st.text_input("Query:", placeholder="What would you like to know about the documentation", key='input')
1166
+ submit_button = st.form_submit_button(label='Send')
1167
+ if submit_button and user_input:
1168
+ output = conversational_chat(user_input)
1169
+ st.session_state['past'].append(user_input)
1170
+ st.session_state['generated'].append(output)
1171
+ user_input = "Query:"
1172
+ #st.session_state['input'] = ""
1173
+ # Display generated responses
1174
+ if st.session_state['generated']:
1175
+ with response_container:
1176
+ for i in range(len(st.session_state['generated'])):
1177
+ message(st.session_state["past"][i], is_user=True, key=str(i) + '_user', avatar_style="shapes")
1178
+ message(st.session_state["generated"][i], key=str(i), avatar_style="icons")
chroma_db_company/a68ce0c8-f49f-46b1-8f8a-0c2f5a9374c1/data_level0.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a13e72541800c513c73dccea69f79e39cf4baef4fa23f7e117c0d6b0f5f99670
3
+ size 3212000
chroma_db_company/a68ce0c8-f49f-46b1-8f8a-0c2f5a9374c1/header.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ec6df10978b056a10062ed99efeef2702fa4a1301fad702b53dd2517103c746
3
+ size 100
chroma_db_company/a68ce0c8-f49f-46b1-8f8a-0c2f5a9374c1/length.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc19b1997119425765295aeab72d76faa6927d4f83985d328c26f20468d6cc76
3
+ size 4000
chroma_db_company/a68ce0c8-f49f-46b1-8f8a-0c2f5a9374c1/link_lists.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
3
+ size 0
chroma_db_company/chroma.sqlite3 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53ceb688eac9d0cb42bbe2b7cb59cada8cd3953d37a6cdb812e3cf7dbe526083
3
+ size 1929216
chroma_db_financial/chroma.sqlite3 ADDED
Binary file (147 kB). View file
 
chroma_db_intercreditor/chroma.sqlite3 ADDED
Binary file (147 kB). View file