Upload 7 files

.chainlit/config.toml

@@ -0,0 +1,120 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Authorized origins
+allow_origins = ["*"]
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Automatically tag threads with the current chat profile (if a chat profile is used)
+auto_tag_thread = true
+
+# Authorize users to spontaneously upload files with messages
+[features.spontaneous_file_upload]
+    enabled = true
+    accept = ["*/*"]
+    max_files = 20
+    max_size_mb = 500
+
+[features.audio]
+    # Threshold for audio recording
+    min_decibels = -45
+    # Delay for the user to start speaking in MS
+    initial_silence_timeout = 3000
+    # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop.
+    silence_timeout = 1500
+    # Above this duration (MS), the recording will forcefully stop.
+    max_duration = 15000
+    # Duration of the audio chunks in MS
+    chunk_duration = 1000
+    # Sample rate of the audio
+    sample_rate = 44100
+
+[UI]
+name = "Shield Chat"
+custom_css = "/public/test.css"
+
+
+# Description of the assistant. This is used for HTML tags.
+# description = ""
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Specify a Javascript file that can be used to customize the user interface.
+# The Javascript file can be served from the public directory.
+# custom_js = "/public/test.js"
+
+# Specify a custom font url.
+# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap"
+
+# Specify a custom meta image url.
+# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
+
+# Specify a custom build directory for the frontend.
+# This can be used to customize the frontend code.
+# Be careful: If this is a relative path, it should not start with a slash.
+# custom_build = "./public/build"
+
+[UI.theme]
+    default = "dark"
+    #layout = "wide"
+    #font_family = "Inter, sans-serif"
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        main = "#F80061"
+        dark = "#980039"
+        light = "#FFE7EB"
+    [UI.theme.light.text]
+        primary = "#212121"
+        secondary = "#616161"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        main = "#F80061"
+        dark = "#980039"
+        light = "#FFE7EB"
+    [UI.theme.dark.text]
+        primary = "#EEEEEE"
+        secondary = "#BDBDBD"
+
+
+[meta]
+generated_by = "1.1.304"

.chainlit/translations/en-US.json

@@ -0,0 +1,229 @@
+{
+    "components": {
+        "atoms": {
+            "buttons": {
+                "userButton": {
+                    "menu": {
+                        "settings": "Settings",
+                        "settingsKey": "S",
+                        "APIKeys": "API Keys",
+                        "logout": "Logout"
+                    }
+                }
+            }
+        },
+        "molecules": {
+            "newChatButton": {
+                "newChat": "New Chat"
+            },
+            "tasklist": {
+                "TaskList": {
+                    "title": "\ud83d\uddd2\ufe0f Task List",
+                    "loading": "Loading...",
+                    "error": "An error occurred"
+                }
+            },
+            "attachments": {
+                "cancelUpload": "Cancel upload",
+                "removeAttachment": "Remove attachment"
+            },
+            "newChatDialog": {
+                "createNewChat": "Create new chat?",
+                "clearChat": "This will clear the current messages and start a new chat.",
+                "cancel": "Cancel",
+                "confirm": "Confirm"
+            },
+            "settingsModal": {
+                "settings": "Settings",
+                "expandMessages": "Expand Messages",
+                "hideChainOfThought": "Hide Chain of Thought",
+                "darkMode": "Dark Mode"
+            },
+            "detailsButton": {
+                "using": "Using",
+                "used": "Used"
+            },
+            "auth": {
+                "authLogin": {
+                    "title": "Login to access the app.",
+                    "form": {
+                        "email": "Email address",
+                        "password": "Password",
+                        "noAccount": "Don't have an account?",
+                        "alreadyHaveAccount": "Already have an account?",
+                        "signup": "Sign Up",
+                        "signin": "Sign In",
+                        "or": "OR",
+                        "continue": "Continue",
+                        "forgotPassword": "Forgot password?",
+                        "passwordMustContain": "Your password must contain:",
+                        "emailRequired": "email is a required field",
+                        "passwordRequired": "password is a required field"
+                    },
+                    "error": {
+                        "default": "Unable to sign in.",
+                        "signin": "Try signing in with a different account.",
+                        "oauthsignin": "Try signing in with a different account.",
+                        "redirect_uri_mismatch": "The redirect URI is not matching the oauth app configuration.",
+                        "oauthcallbackerror": "Try signing in with a different account.",
+                        "oauthcreateaccount": "Try signing in with a different account.",
+                        "emailcreateaccount": "Try signing in with a different account.",
+                        "callback": "Try signing in with a different account.",
+                        "oauthaccountnotlinked": "To confirm your identity, sign in with the same account you used originally.",
+                        "emailsignin": "The e-mail could not be sent.",
+                        "emailverify": "Please verify your email, a new email has been sent.",
+                        "credentialssignin": "Sign in failed. Check the details you provided are correct.",
+                        "sessionrequired": "Please sign in to access this page."
+                    }
+                },
+                "authVerifyEmail": {
+                    "almostThere": "You're almost there! We've sent an email to ",
+                    "verifyEmailLink": "Please click on the link in that email to complete your signup.",
+                    "didNotReceive": "Can't find the email?",
+                    "resendEmail": "Resend email",
+                    "goBack": "Go Back",
+                    "emailSent": "Email sent successfully.",
+                    "verifyEmail": "Verify your email address"
+                },
+                "providerButton": {
+                    "continue": "Continue with {{provider}}",
+                    "signup": "Sign up with {{provider}}"
+                },
+                "authResetPassword": {
+                    "newPasswordRequired": "New password is a required field",
+                    "passwordsMustMatch": "Passwords must match",
+                    "confirmPasswordRequired": "Confirm password is a required field",
+                    "newPassword": "New password",
+                    "confirmPassword": "Confirm password",
+                    "resetPassword": "Reset Password"
+                },
+                "authForgotPassword": {
+                    "email": "Email address",
+                    "emailRequired": "email is a required field",
+                    "emailSent": "Please check the email address {{email}} for instructions to reset your password.",
+                    "enterEmail": "Enter your email address and we will send you instructions to reset your password.",
+                    "resendEmail": "Resend email",
+                    "continue": "Continue",
+                    "goBack": "Go Back"
+                }
+            }
+        },
+        "organisms": {
+            "chat": {
+                "history": {
+                    "index": {
+                        "showHistory": "Show history",
+                        "lastInputs": "Last Inputs",
+                        "noInputs": "Such empty...",
+                        "loading": "Loading..."
+                    }
+                },
+                "inputBox": {
+                    "input": {
+                        "placeholder": "Type your message here..."
+                    },
+                    "speechButton": {
+                        "start": "Start recording",
+                        "stop": "Stop recording"
+                    },
+                    "SubmitButton": {
+                        "sendMessage": "Send message",
+                        "stopTask": "Stop Task"
+                    },
+                    "UploadButton": {
+                        "attachFiles": "Attach files"
+                    },
+                    "waterMark": {
+                        "text": "Built with"
+                    }
+                },
+                "Messages": {
+                    "index": {
+                        "running": "Running",
+                        "executedSuccessfully": "executed successfully",
+                        "failed": "failed",
+                        "feedbackUpdated": "Feedback updated",
+                        "updating": "Updating"
+                    }
+                },
+                "dropScreen": {
+                    "dropYourFilesHere": "Drop your files here"
+                },
+                "index": {
+                    "failedToUpload": "Failed to upload",
+                    "cancelledUploadOf": "Cancelled upload of",
+                    "couldNotReachServer": "Could not reach the server",
+                    "continuingChat": "Continuing previous chat"
+                },
+                "settings": {
+                    "settingsPanel": "Settings panel",
+                    "reset": "Reset",
+                    "cancel": "Cancel",
+                    "confirm": "Confirm"
+                }
+            },
+            "threadHistory": {
+                "sidebar": {
+                    "filters": {
+                        "FeedbackSelect": {
+                            "feedbackAll": "Feedback: All",
+                            "feedbackPositive": "Feedback: Positive",
+                            "feedbackNegative": "Feedback: Negative"
+                        },
+                        "SearchBar": {
+                            "search": "Search"
+                        }
+                    },
+                    "DeleteThreadButton": {
+                        "confirmMessage": "This will delete the thread as well as it's messages and elements.",
+                        "cancel": "Cancel",
+                        "confirm": "Confirm",
+                        "deletingChat": "Deleting chat",
+                        "chatDeleted": "Chat deleted"
+                    },
+                    "index": {
+                        "pastChats": "Past Chats"
+                    },
+                    "ThreadList": {
+                        "empty": "Empty...",
+                        "today": "Today",
+                        "yesterday": "Yesterday",
+                        "previous7days": "Previous 7 days",
+                        "previous30days": "Previous 30 days"
+                    },
+                    "TriggerButton": {
+                        "closeSidebar": "Close sidebar",
+                        "openSidebar": "Open sidebar"
+                    }
+                },
+                "Thread": {
+                    "backToChat": "Go back to chat",
+                    "chatCreatedOn": "This chat was created on"
+                }
+            },
+            "header": {
+                "chat": "Chat",
+                "readme": "Readme"
+            }
+        }
+    },
+    "hooks": {
+        "useLLMProviders": {
+            "failedToFetchProviders": "Failed to fetch providers:"
+        }
+    },
+    "pages": {
+        "Design": {},
+        "Env": {
+            "savedSuccessfully": "Saved successfully",
+            "requiredApiKeys": "Required API Keys",
+            "requiredApiKeysInfo": "To use this app, the following API keys are required. The keys are stored on your device's local storage."
+        },
+        "Page": {
+            "notPartOfProject": "You are not part of this project."
+        },
+        "ResumeButton": {
+            "resumeChat": "Resume Chat"
+        }
+    }
+}

.env

@@ -0,0 +1 @@
+GROQ_API_KEY=gsk_your_key

app.py

@@ -0,0 +1,196 @@
+import re
+import PyPDF2
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from langchain.chains import ConversationalRetrievalChain
+from langchain_community.chat_models import ChatOllama
+from langchain_groq import ChatGroq
+from langchain.memory import ChatMessageHistory, ConversationBufferMemory
+import chainlit as cl
+from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
+import logging
+import pypandoc
+import pdfkit
+from paddleocr import PaddleOCR
+import fitz  
+import asyncio
+
+llm_groq = ChatGroq(
+            model_name='llama3-70b-8192'
+    )
+
+# Initialize anonymizer
+anonymizer = PresidioReversibleAnonymizer(analyzed_fields=['PERSON', 'EMAIL_ADDRESS', 'PHONE_NUMBER', 'IBAN_CODE', 'CREDIT_CARD', 'CRYPTO', 'IP_ADDRESS', 'LOCATION', 'DATE_TIME', 'NRP', 'MEDICAL_LICENSE', 'URL', 'US_BANK_NUMBER', 'US_DRIVER_LICENSE', 'US_ITIN', 'US_PASSPORT', 'US_SSN'], faker_seed=18)
+
+def extract_text_from_pdf(file_path):
+    pdf = PyPDF2.PdfReader(file_path)
+    pdf_text = ""
+    for page in pdf.pages:
+        pdf_text += page.extract_text()
+    return pdf_text
+
+def has_sufficient_selectable_text(page, threshold=50):
+    text = page.extract_text()
+    if len(text.strip()) > threshold:
+        return True
+    return False
+
+async def get_text(file_path):
+    text = ""
+    try:
+        logging.info("Starting OCR process for file: %s", file_path)
+        extension = file_path.split(".")[-1].lower()
+        allowed_extension = ["jpg", "jpeg", "png", "pdf", "docx"]
+        if extension not in allowed_extension:
+            error = "Not a valid File. Allowed Format are jpg, jpeg, png, pdf, docx"
+            logging.error(error)
+            return {"error": error}
+        
+        if extension == "docx":
+            file_path = convert_docx_to_pdf(file_path)
+        
+        ocr = PaddleOCR(use_angle_cls=True, lang='en')
+        result = ocr.ocr(file_path, cls=True)
+        for idx in range(len(result)):
+            res = result[idx]
+            for line in res:
+                text += line[1][0] + " "
+        logging.info("OCR process completed successfully for file: %s", file_path)
+    except Exception as e:
+        logging.error("Error occurred during OCR process for file %s: %s", file_path, e)
+        text = "Error occurred during OCR process."
+    logging.info("Extracted text: %s", text)
+    return text
+
+def convert_docx_to_pdf(input_path):
+    html_path = input_path.replace('.docx', '.html')
+    output_path = ".".join(input_path.split(".")[:-1]) + ".pdf"
+    pypandoc.convert_file(input_path, 'html', outputfile=html_path)
+    pdfkit.from_file(html_path, output_path)
+    logging.info("DOCX Format Handled")
+    return output_path
+
+async def extract_text_from_mixed_pdf(file_path):
+    pdf = PyPDF2.PdfReader(file_path)
+    ocr = PaddleOCR(use_angle_cls=True, lang='en')
+    pdf_text = ""
+    for i, page in enumerate(pdf.pages):
+        text = page.extract_text()
+        if not has_sufficient_selectable_text(page):
+            logging.info(f"Page {i+1} has insufficient selectable text, performing OCR.")
+            pdf_document = fitz.open(file_path)
+            pdf_page = pdf_document.load_page(i)
+            pix = pdf_page.get_pixmap()
+            image_path = f"page_{i+1}.png"
+            pix.save(image_path)
+            result = ocr.ocr(image_path, cls=True)
+            for idx in range(len(result)):
+                res = result[idx]
+                for line in res:
+                    text += line[1][0] + " "
+        pdf_text += text
+    return pdf_text
+
+@cl.on_chat_start
+async def on_chat_start():
+    
+    files = None # Initialize variable to store uploaded files
+
+    # Wait for the user to upload a file
+    while files is None:
+        files = await cl.AskFileMessage(
+            content="Please upload a pdf file to begin!",
+            # accept=["application/pdf"],
+            accept=["application/pdf", "image/jpeg", "image/png", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
+            max_size_mb=100,
+            timeout=180, 
+        ).send()
+
+    file = files[0] # Get the first uploaded file
+    
+    # Inform the user that processing has started
+    msg = cl.Message(content=f"Processing `{file.name}`...")
+    await msg.send()
+
+    # Extract text from PDF, checking for selectable and handwritten text
+    if file.name.endswith('.pdf'):
+        pdf_text = await extract_text_from_mixed_pdf(file.path)
+    else:
+        pdf_text = await get_text(file.path)
+
+    # Anonymize the text
+    anonymized_text = anonymizer.anonymize(
+        pdf_text
+    )
+    
+    # with splitting into chunks
+    # {
+    # # Split the sanitized text into chunks
+    # text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    # texts = text_splitter.split_text(anonymized_text)
+
+    # # Create metadata for each chunk
+    # metadatas = [{"source": f"{i}-pl"} for i in range(len(texts))]
+
+    # # Create a Chroma vector store
+    # embeddings = OllamaEmbeddings(model="nomic-embed-text")
+    # docsearch = await cl.make_async(Chroma.from_texts)(
+    #     texts, embeddings, metadatas=metadatas
+    # )
+    # }
+    
+    # without splitting into chunks
+    # {
+    # Create a Chroma vector store
+    embeddings = OllamaEmbeddings(model="nomic-embed-text")
+    docsearch = await cl.make_async(Chroma.from_texts)(
+        [anonymized_text], embeddings, metadatas=[{"source": "0-pl"}]
+    )
+    # }
+    
+    # Initialize message history for conversation
+    message_history = ChatMessageHistory()
+    
+    # Memory for conversational context
+    memory = ConversationBufferMemory(
+        memory_key="chat_history",
+        output_key="answer",
+        chat_memory=message_history,
+        return_messages=True,
+    )
+
+    # Create a chain that uses the Chroma vector store
+    chain = ConversationalRetrievalChain.from_llm(
+        llm = llm_groq,
+        chain_type="stuff",
+        retriever=docsearch.as_retriever(),
+        memory=memory,
+        return_source_documents=True,
+    )
+
+    # Let the user know that the system is ready
+    msg.content = f"Processing `{file.name}` done. You can now ask questions!"
+    await msg.update()
+    # Store the chain in user session
+    cl.user_session.set("chain", chain)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+        
+    # Retrieve the chain from user session
+    chain = cl.user_session.get("chain") 
+    # Callbacks happen asynchronously/parallel 
+    cb = cl.AsyncLangchainCallbackHandler()
+    
+    # Call the chain with user's message content
+    res = await chain.ainvoke(message.content, callbacks=[cb])
+    answer = anonymizer.deanonymize(
+        "ok"+res["answer"]
+    )  
+    text_elements = [] 
+            
+    # Return results
+    await cl.Message(content=answer, elements=text_elements).send()
+

chainlit.md

@@ -0,0 +1 @@
+Developed by "Akash Bhavsar"

public/test.css

@@ -0,0 +1,20 @@
+.MuiButtonBase-root.MuiIconButton-root.MuiIconButton-sizeMedium.css-1egpgfe {
+  display: none;
+}
+
+.MuiStack-root.watermark.css-1705j0v {
+  display: none;
+}
+
+.MuiAvatar-img.css-1hy9t21 {
+  content: url("/public/image.png"); /* Path to your custom avatar image */
+}
+
+img[src="http://localhost:8000/logo?theme=dark"]
+{
+  display: none;
+}
+
+#open-sidebar-button {
+  display: none;
+}

requirements.txt

@@ -0,0 +1,20 @@
+chainlitchainlit==1.1.304
+langchain==0.2.5
+langchain-community==0.2.5
+langchain-core==0.2.9
+langchain-groq==0.1.5
+langchain-experimental==0.0.61
+PyPDF2==3.0.1
+chromadb==0.5.3
+groq==0.9.0
+ollama==0.2.1
+pypandoc==1.13
+pdfkit==1.0.0
+docx2pdf==0.1.8
+paddlepaddle==2.6.1
+paddleocr==2.7.3
+presidio-analyzer==2.2.354
+presidio-anonymizer==2.2.354
+spacy==3.7.5
+Faker==25.9.1
+# python 3.10.0