Spaces:
Runtime error
Runtime error
Updated pdf parsing
Browse fileshttps://huggingface.co/spaces/fffiloni/langchain-chat-with-pdf/blob/main/app.py
- pdftoqa_generator.py +20 -18
pdftoqa_generator.py
CHANGED
@@ -5,7 +5,7 @@ import statistics
|
|
5 |
|
6 |
import gradio as gr
|
7 |
import pandas as pd
|
8 |
-
from langchain.document_loaders import
|
9 |
from langchain.text_splitter import (
|
10 |
CharacterTextSplitter,
|
11 |
RecursiveCharacterTextSplitter,
|
@@ -18,28 +18,30 @@ os.environ["OPENAI_API_KEY"] = "sk-"
|
|
18 |
|
19 |
|
20 |
def pdf_parser(uploaded_file):
|
|
|
21 |
bytes_data = uploaded_file.read()
|
22 |
with NamedTemporaryFile(delete=False) as tmp: # open a named temporary file
|
23 |
tmp.write(bytes_data) # Write data from the uploaded file into it
|
24 |
pdf_loader = PyPDFLoader(tmp.name) # <---- now it works!
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
documents = pdf_loader.load()
|
29 |
-
documents_text = [d.page_content for d in documents]
|
30 |
-
|
31 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
32 |
-
# Set a really small chunk size, just to show.
|
33 |
-
chunk_size=600,
|
34 |
-
chunk_overlap=200,
|
35 |
-
length_function=len,
|
36 |
-
is_separator_regex=False,
|
37 |
-
)
|
38 |
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
|
44 |
|
45 |
def qa_generator(texts):
|
|
|
5 |
|
6 |
import gradio as gr
|
7 |
import pandas as pd
|
8 |
+
from langchain.document_loaders import OnlinePDFLoader
|
9 |
from langchain.text_splitter import (
|
10 |
CharacterTextSplitter,
|
11 |
RecursiveCharacterTextSplitter,
|
|
|
18 |
|
19 |
|
20 |
def pdf_parser(uploaded_file):
|
21 |
+
'''
|
22 |
bytes_data = uploaded_file.read()
|
23 |
with NamedTemporaryFile(delete=False) as tmp: # open a named temporary file
|
24 |
tmp.write(bytes_data) # Write data from the uploaded file into it
|
25 |
pdf_loader = PyPDFLoader(tmp.name) # <---- now it works!
|
26 |
+
'''
|
27 |
+
#pdf_loader = PyPDFLoader(file_path) only for file path offline
|
28 |
+
pdf_loader=OnlinePDFLoader(uploaded_file.name) #https://huggingface.co/spaces/fffiloni/langchain-chat-with-pdf/blob/main/app.py
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
29 |
|
30 |
+
documents = pdf_loader.load()
|
31 |
+
documents_text = [d.page_content for d in documents]
|
32 |
+
|
33 |
+
text_splitter = RecursiveCharacterTextSplitter(
|
34 |
+
# Set a really small chunk size, just to show.
|
35 |
+
chunk_size=600,
|
36 |
+
chunk_overlap=200,
|
37 |
+
length_function=len,
|
38 |
+
is_separator_regex=False,
|
39 |
+
)
|
40 |
+
|
41 |
+
# Split the text into chunks
|
42 |
+
texts = text_splitter.create_documents(documents_text)
|
43 |
+
#os.remove(tmp.name) # remove temp file
|
44 |
+
return texts
|
45 |
|
46 |
|
47 |
def qa_generator(texts):
|