feat: adicionando llama_parse e algumas tipagens

example.env → .env.example

@@ -5,4 +5,5 @@ HUGGINGFACEHUB_API_TOKEN=""
 LANGCHAIN_API_KEY=""
 CLAUDE_API_KEY=""
 COHERE_API_KEY=""
-BUBBLE_TOKEN=""
+BUBBLE_TOKEN=""
+LLAMA_CLOUD_API_KEY=""

_utils/bubble_integrations/obter_arquivo.py

@@ -16,8 +16,8 @@ headers = {"Authorization": f"Bearer {os.environ.get("BUBBLE_TOKEN")}"}
 
 def get_pdf_from_bubble(
     file_url=f"https://vella.app.br/version-test/fileupload/f1735864316650x718601440484441900/Boleto_DIGITICS%20Servic%CC%A7os%20de%20Secretariado%20LTDA_30_12_2024_804841714.pdf",
-    bubble_editor_version="version-test",
 ):
+
     result = PyPDFLoader(file_url, headers=headers)
 
-    return result
+    return result.load()

_utils/gerar_relatorio_modelo_usuario/contextual_retriever.py

@@ -6,6 +6,7 @@ from _utils.gerar_relatorio_modelo_usuario.prompts import (
 )
 from _utils.bubble_integrations.obter_arquivo import get_pdf_from_bubble
 from _utils.chains.Chain_class import Chain
+from _utils.handle_files import return_document_list_with_llama_parser
 from _utils.prompts.Prompt_class import Prompt
 from _utils.splitters.Splitter_class import Splitter
 from setup.easy_imports import PyPDFLoader
@@ -14,6 +15,7 @@ from typing import List, Dict, Tuple, Optional
 from anthropic import Anthropic, AsyncAnthropic
 import logging
 from langchain.schema import Document
+from llama_index import Document as Llama_Index_Document
 import asyncio
 from langchain.prompts import PromptTemplate
 from typing import List
@@ -142,26 +144,30 @@ class ContextualRetriever:
         return contextualized_chunks
 
 
-def get_full_text_and_all_PDFs_chunks(contexto, listaPDFs, splitterObject: Splitter):
+async def get_full_text_and_all_PDFs_chunks(
+    listaPDFs: List[str],
+    splitterObject: Splitter,
+    should_use_llama_parse: bool,
+):
     all_PDFs_chunks = []
-    full_text = ""
-    if contexto:
-        full_text = contexto
-        chunks = splitterObject.load_and_split_text(full_text)
-        all_PDFs_chunks = chunks
-    else:
-        # Load and process document
-        for pdf in listaPDFs:
-            pdf_path = pdf
-            chunks = splitterObject.load_and_split_document(pdf_path)
-            all_PDFs_chunks = all_PDFs_chunks + chunks
-        # Get full text for contextualization
-        # loader = PyPDFLoader(pdf_path)
-        loader = get_pdf_from_bubble(pdf_path)
-        pages = loader.load()
-        full_text = " ".join([page.page_content for page in pages])
-
-    return full_text, all_PDFs_chunks, pages
+
+    pages: List[Document] = []
+
+    # Load and process document
+    for pdf_path in listaPDFs:
+        if should_use_llama_parse:
+            pages = pages + await return_document_list_with_llama_parser(pdf_path)
+        else:
+            pages = pages + get_pdf_from_bubble(pdf_path)
+        chunks = splitterObject.load_and_split_document(pdf_path, pages)
+        all_PDFs_chunks = all_PDFs_chunks + chunks
+    # Get full text for contextualization
+    # loader = PyPDFLoader(pdf_path)
+
+    # full_text = ""
+    # full_text = " ".join([page.page_content for page in pages])
+
+    return all_PDFs_chunks, pages  # , full_text
 
 
 async def contextualize_chunk_based_on_serializer(

_utils/gerar_relatorio_modelo_usuario/llm_calls.py

@@ -1,4 +1,6 @@
 import os
+
+from pydantic import SecretStr
 from setup.environment import default_model
 from langchain_core.messages import HumanMessage
 from langchain_openai import ChatOpenAI
@@ -17,14 +19,16 @@ async def aclaude_answer(claude_client, claude_context_model, prompt):
 
 
 async def agpt_answer(prompt):
-    gpt = ChatOpenAI(
-        temperature=0,
-        model="gpt-4o-mini",
-        api_key=os.environ.get("OPENAI_API_KEY"),
-        max_retries=5,
-    )
-    response = await gpt.ainvoke([HumanMessage(content=prompt)])
-    return response.content
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if api_key:
+        gpt = ChatOpenAI(
+            temperature=0,
+            model="gpt-4o-mini",
+            api_key=SecretStr(api_key),
+            max_retries=5,
+        )
+        response = await gpt.ainvoke([HumanMessage(content=prompt)])
+        return response.content
 
 
 def gpt_answer(
@@ -34,14 +38,16 @@ def gpt_answer(
     max_retries=5,
     shouldReturnFullResponse=False,
 ):
-    gpt = ChatOpenAI(
-        temperature=temperature,
-        model=model,
-        api_key=os.environ.get("OPENAI_API_KEY"),
-        max_retries=max_retries,
-    )
-    response = gpt.invoke([HumanMessage(content=prompt)])
-    if shouldReturnFullResponse:
-        return response
-    else:
-        return response.content
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if api_key:
+        gpt = ChatOpenAI(
+            temperature=temperature,
+            model=model,
+            api_key=SecretStr(api_key),
+            max_retries=max_retries,
+        )
+        response = gpt.invoke([HumanMessage(content=prompt)])
+        if shouldReturnFullResponse:
+            return response
+        else:
+            return response.content

_utils/handle_files.py

@@ -1,6 +1,9 @@
 import tempfile, os
-
+from typing import List
+from langchain_core.documents import Document as LangchainDocument
+from llama_index import Document
 from _utils.bubble_integrations.obter_arquivo import get_pdf_from_bubble
+from llama_parse import LlamaParse, ResultType
 
 
 def handle_pdf_files_from_serializer(files):
@@ -19,6 +22,37 @@ def handle_pdf_files_from_serializer(files):
     print("listaPDFs: ", listaPDFs)
     return listaPDFs
 
+
 def remove_pdf_temp_files(listaPDFs):
     for file in listaPDFs:
         os.remove(file)
+
+
+async def return_document_list_with_llama_parser(file: str):
+    llama_parser_api = os.getenv("LLAMA_CLOUD_API_KEY")
+    documents: List[LangchainDocument] = []
+    if llama_parser_api:
+        parser = LlamaParse(
+            api_key=llama_parser_api,
+            result_type=ResultType.JSON,  # Options: 'text', 'markdown', 'json', 'structured'
+            language="pt",
+            verbose=True,
+        )
+
+        parsed_document = await parser.aget_json(file)
+        for doc in parsed_document[0].get("pages"):  # type: ignore
+            # documents.append(doc.to_langchain_format())
+
+            langchain_document = LangchainDocument(
+                page_content=doc.get("md"),  # type: ignore
+                metadata={
+                    "page": doc.get("page"),  # type: ignore
+                    # **doc.get("metadata", {}),  # type: ignore
+                },  # Include page number in metadata
+            )
+
+            documents.append(langchain_document)
+
+        return documents
+    else:
+        raise ValueError("Não foi possível obter a API_KEY do llama parser")

_utils/resumo_completo_cursor.py

@@ -38,9 +38,7 @@ os.environ.get("LANGCHAIN_API_KEY")
 os.environ["LANGCHAIN_PROJECT"] = "VELLA"
 
 
-async def get_llm_summary_answer_by_cursor_complete(
-    serializer, listaPDFs=None, contexto=None
-):
+async def get_llm_summary_answer_by_cursor_complete(serializer, listaPDFs=None):
     """Parâmetro "contexto" só deve ser passado quando quiser utilizar o teste com ragas, e assim, não quiser passar PDFs"""
     # Configuration
     config = RetrievalConfig(
@@ -70,8 +68,8 @@ async def get_llm_summary_answer_by_cursor_complete(
         reciprocal_rank_fusion=reciprocal_rank_fusion,
     )
 
-    full_text, allPdfsChunks, pages = get_full_text_and_all_PDFs_chunks(
-        contexto, listaPDFs, summarizer.splitter
+    allPdfsChunks, pages = await get_full_text_and_all_PDFs_chunks(
+        listaPDFs, summarizer.splitter, serializer["should_use_llama_parse"]
     )
 
     chunks_passados, is_contextualized_chunk = (

_utils/splitters/Splitter_class.py

@@ -1,6 +1,6 @@
 from _utils.bubble_integrations.obter_arquivo import get_pdf_from_bubble
 from setup.easy_imports import PyPDFLoader, RecursiveCharacterTextSplitter, Document
-from typing import List, Dict, Tuple, Optional
+from typing import List, Dict, Tuple, Optional, cast
 from _utils.models.gerar_relatorio import (
     DocumentChunk,
 )
@@ -18,13 +18,15 @@ class Splitter:
         )
         self.chunk_metadata = {}  # Store chunk metadata for tracing
 
-    def load_and_split_document(self, pdf_path: str) -> List[DocumentChunk]:
+    def load_and_split_document(
+        self, pdf_path: str, pages: List[Document] | None
+    ) -> List[DocumentChunk]:
         """Load PDF and split into chunks with metadata"""
         # loader = PyPDFLoader(pdf_path)
-        loader = get_pdf_from_bubble(pdf_path)
-        pages = (
-            loader.load()
-        )  # Gera uma lista de objetos Document, sendo cada item da lista referente a UMA PÁGINA inteira do PDF.
+        if not pages:
+            pages = get_pdf_from_bubble(
+                pdf_path
+            )  # Gera uma lista de objetos Document, sendo cada item da lista referente a UMA PÁGINA inteira do PDF.
         chunks = []
         char_count = 0
 
@@ -43,7 +45,8 @@ class Splitter:
 
                 doc_chunk = DocumentChunk(  # Gera o objeto do chunk com informações adicionais, como a posição e id do chunk
                     content=chunk,
-                    page_number=page.metadata.get("page") + 1,  # 1-based page numbering
+                    page_number=cast(int, page.metadata.get("page"))
+                    + 1,  # 1-based page numbering
                     chunk_id=chunk_id,
                     start_char=char_count + start_char,
                     end_char=char_count + end_char,
@@ -71,8 +74,7 @@ class Splitter:
         page_chunks = self.text_splitter.split_text(
             text
         )  # Quebra o item que é um Document de UMA PÁGINA inteira em um lista onde cada item é referente a um chunk, que são pedaços menores do que uma página.
-        print("\n\n\n")
-        print("page_chunks: ", page_chunks)
+        print("\n\n\npage_chunks: ", page_chunks)
 
         for chunk in page_chunks:
             chunk_id = str(uuid.uuid4())
@@ -83,7 +85,8 @@ class Splitter:
 
             doc_chunk = DocumentChunk(  # Gera o objeto do chunk com informações adicionais, como a posição e id do chunk
                 content=chunk,
-                page_number=page.metadata.get("page") + 1,  # 1-based page numbering
+                page_number=cast(int, page.metadata.get("page"))
+                + 1,  # 1-based page numbering
                 chunk_id=chunk_id,
                 start_char=char_count + start_char,
                 end_char=char_count + end_char,

gerar_documento/serializer.py

@@ -44,7 +44,7 @@ class GerarDocumentoSerializer(ResumoCursorSerializer):
     )
     gpt_temperature = serializers.FloatField(default=0)
     id_modelo_do_usuario = serializers.IntegerField(required=False)
-    should_have_contextual_chunks = serializers.BooleanField(default=False)
+    should_have_contextual_chunks = serializers.BooleanField(default=False)  # type: ignore
 
 
 class GerarDocumentoComPDFProprioSerializer(ResumoCursorSerializer):
@@ -71,4 +71,5 @@ class GerarDocumentoComPDFProprioSerializer(ResumoCursorSerializer):
     )
     gpt_temperature = serializers.FloatField(default=0)
     id_modelo_do_usuario = serializers.IntegerField(required=False, default=11)
-    should_have_contextual_chunks = serializers.BooleanField(default=False)
+    should_have_contextual_chunks = serializers.BooleanField(default=False)  # type: ignore
+    should_use_llama_parse = serializers.BooleanField(required=False, default=True)  # type: ignore

gerar_documento/views.py

@@ -1,3 +1,4 @@
+from typing import Any, Dict, cast
 from setup.easy_imports import (
     Response,
     AsyncAPIView,
@@ -27,7 +28,9 @@ class GerarDocumentoView(AsyncAPIView):
         print(f"\n\nDATA E HORA DA REQUISIÇÃO: {datetime.now()}")
         serializer = GerarDocumentoSerializer(data=request.data)
         if serializer.is_valid(raise_exception=True):
-            data = serializer.validated_data
+            if serializer.validated_data:
+                raise ValueError("Erro no validated_data")
+            data = cast(Dict[str, Any], serializer.validated_data)
             print("\n\ndata: ", data)
 
             data["prompt_auxiliar"] = (
@@ -59,7 +62,7 @@ class GerarDocumentoComPDFProprioView(AsyncAPIView):
         print(f"\n\nDATA E HORA DA REQUISIÇÃO: {datetime.now()}")
         serializer = GerarDocumentoComPDFProprioSerializer(data=request.data)
         if serializer.is_valid(raise_exception=True):
-            data = serializer.validated_data
+            data = cast(Dict[str, Any], serializer.validated_data)
             print("\n\ndata: ", data)
 
             listaPDFs = handle_pdf_files_from_serializer(data["files"])