Changed app space image, removed unnecessary files

README.md

@@ -1,6 +1,6 @@
 ---
 title: Light PDF web QA chatbot
-emoji: 📈
+emoji: 🌍
 colorFrom: yellow
 colorTo: yellow
 sdk: gradio

chatfuncs/.ipynb_checkpoints/chatfuncs-checkpoint.py

@@ -1,553 +0,0 @@
-# ---
-# jupyter:
-#   jupytext:
-#     formats: ipynb,py:light
-#     text_representation:
-#       extension: .py
-#       format_name: light
-#       format_version: '1.5'
-#       jupytext_version: 1.14.6
-#   kernelspec:
-#     display_name: Python 3 (ipykernel)
-#     language: python
-#     name: python3
-# ---
-
-# +
-import os
-import datetime
-from typing import Dict, List, Tuple
-from itertools import compress
-import pandas as pd
-
-from langchain import PromptTemplate
-from langchain.chains import LLMChain
-from langchain.chains.base import Chain
-from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
-from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings
-from langchain.chains.qa_with_sources import load_qa_with_sources_chain
-from langchain.prompts import PromptTemplate
-from langchain.retrievers import TFIDFRetriever, SVMRetriever
-from langchain.vectorstores import FAISS
-from langchain.llms import HuggingFacePipeline
-
-from pydantic import BaseModel
-
-import nltk
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
-
-import torch
-#from transformers import pipeline
-from optimum.pipelines import pipeline
-from transformers import AutoTokenizer, TextStreamer, AutoModelForSeq2SeqLM, TextIteratorStreamer
-from threading import Thread
-
-import gradio as gr
-
-
-# -
-
-# # Pre-load stopwords, vectorstore, models
-
-# +
-def get_faiss_store(faiss_vstore_folder,embeddings):
-    import zipfile
-    with zipfile.ZipFile(faiss_vstore_folder + '/faiss_lambeth_census_embedding.zip', 'r') as zip_ref:
-        zip_ref.extractall(faiss_vstore_folder)
-
-    faiss_vstore = FAISS.load_local(folder_path=faiss_vstore_folder, embeddings=embeddings)
-    os.remove(faiss_vstore_folder + "/index.faiss")
-    os.remove(faiss_vstore_folder + "/index.pkl")
-    
-    return faiss_vstore
-
-#def set_hf_api_key(api_key, chain_agent):
-    #if api_key:
-       #os.environ["HUGGINGFACEHUB_API_TOKEN"] = api_key
-        #vectorstore = get_faiss_store(faiss_vstore_folder="faiss_lambeth_census_embedding.zip",embeddings=embeddings)
-        #qa_chain = create_prompt_templates(vectorstore)
-        #print(qa_chain)
-        #os.environ["HUGGINGFACEHUB_API_TOKEN"] = ""
-        #return qa_chain
-
-
-# -
-
-def create_hf_model(model_name = "declare-lab/flan-alpaca-large"):
-
-    model_id = model_name
-    torch_device = "cuda" if torch.cuda.is_available() else "cpu"
-    print("Running on device:", torch_device)
-    print("CPU threads:", torch.get_num_threads())
-    
-
-
-    if torch_device == "cuda":
-        model = AutoModelForSeq2SeqLM.from_pretrained(model_id, load_in_8bit=True, device_map="auto")
-    else:
-        #torch.set_num_threads(8)
-        model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
-    tokenizer = AutoTokenizer.from_pretrained(model_id)
-    
-    return model, tokenizer, torch_device
-
-# +
-# Add some stopwords to nltk default
-
-nltk.download('stopwords')
-stopwords = nltk.corpus.stopwords.words('english')
-#print(stopwords.words('english'))
-newStopWords = ['what','how', 'when', 'which', 'who', 'change', 'changed', 'do', 'did', 'increase', 'decrease', 'increased',
-                'decreased', 'proportion', 'percentage', 'report', 'reporting','say', 'said']
-stopwords.extend(newStopWords)
-# -
-
-# Embeddings
-#model_name = "sentence-transformers/all-MiniLM-L6-v2"
-#embeddings = HuggingFaceEmbeddings(model_name=model_name)
-embed_model_name = "hkunlp/instructor-large"
-embeddings = HuggingFaceInstructEmbeddings(model_name=embed_model_name)
-vectorstore = get_faiss_store(faiss_vstore_folder="faiss_lambeth_census_embedding",embeddings=embeddings)
-
-# +
-# Models
-
-#checkpoint = 'declare-lab/flan-alpaca-base' # Flan Alpaca Base incorrectly interprets text based on input (e.g. if you use words like increase or decrease in the question it will respond falsely often). Flan Alpaca Large is much more consistent
-checkpoint = 'declare-lab/flan-alpaca-large'
-
-model, tokenizer, torch_device = create_hf_model(model_name = checkpoint)
-
-
-# Look at this for streaming text with huggingface and langchain (last example): https://github.com/hwchase17/langchain/issues/2918
-
-streamer = TextStreamer(tokenizer, skip_prompt=True)
-
-pipe = pipeline('text2text-generation', 
-                 model = checkpoint,
-#                tokenizer = tokenizer,
-                 max_length=512, 
-                 #do_sample=True,
-                 temperature=0.000001,
-                 #top_p=0.95,
-                 #repetition_penalty=1.15,
-                 accelerator="bettertransformer",
-                 streamer=streamer
-                )
-
-checkpoint_keywords = 'ml6team/keyphrase-generation-t5-small-inspec'
-
-keyword_model = pipeline('text2text-generation', 
-                 model = checkpoint_keywords,
-                 accelerator="bettertransformer"
-                )
-
-
-# -
-
-# # Chat history
-
-def clear_chat(chat_history_state, sources, chat_message):
-    chat_history_state = []
-    sources = ''
-    chat_message = ''
-    return chat_history_state, sources, chat_message
-
-
-def _get_chat_history(chat_history: List[Tuple[str, str]]): # Limit to last 3 interactions only
-    max_chat_length = 3
-
-    if len(chat_history) > max_chat_length:
-        chat_history = chat_history[-max_chat_length:]
-        
-    print(chat_history)
-
-    first_q = ""
-    for human_s, ai_s in chat_history:
-        first_q = human_s
-        break
-
-    conversation = ""
-    for human_s, ai_s in chat_history:
-        human = f"Human: " + human_s
-        ai = f"Assistant: " + ai_s
-        conversation += "\n" + "\n".join([human, ai])
-
-    return conversation, first_q
-
-
-def adapt_q_from_chat_history(keyword_model, new_question_keywords, question, chat_history):
-        t5_small_keyphrase = HuggingFacePipeline(pipeline=keyword_model)
-        memory_llm = t5_small_keyphrase#flan_alpaca#flan_t5_xxl
-        new_q_memory_llm = t5_small_keyphrase#flan_alpaca#flan_t5_xxl
-    
-
-        memory_prompt = PromptTemplate(
-            template = "{chat_history_first_q}",
-            input_variables=["chat_history_first_q"]
-        )
-            #template = "Extract the names of people, things, or places from the following text: {chat_history}",#\n    Original question: {question}\n  New list:",
-            #template = "Extract keywords, and the names of people or places from the following text: {chat_history}",#\n    Original question: {question}\n  New list:",
-            #\n    Original question: {question}\n  New list:",
-        
-
-            #example_prompt=_eg_prompt,
-            #input_variables=["question", "chat_history"]
-            #input_variables=["chat_history"]
-        
-        memory_extractor = LLMChain(llm=memory_llm, prompt=memory_prompt)
-       
-        #new_question_keywords = #remove_stopwords(question)
-
-        print("new_question_keywords:")
-        print(new_question_keywords)
-
-        chat_history_str, chat_history_first_q = _get_chat_history(chat_history)
-        if chat_history_str:
-            
-            extracted_memory = memory_extractor.run(
-                chat_history_first_q=chat_history_first_q # question=question, chat_history=chat_history_str, 
-            )
-            
-            new_question_kworded = extracted_memory + " " + new_question_keywords
-            new_question = extracted_memory + " " + question
-            
-        else:
-            new_question = question
-            new_question_kworded = new_question_keywords
-            
-        return new_question, new_question_kworded
-
-
-# # Prompt creation
-
-def remove_q_stopwords(question):
-    # Prepare question by removing keywords
-    text = question.lower()
-    text_tokens = word_tokenize(text)
-    tokens_without_sw = [word for word in text_tokens if not word in stopwords]
-    new_question_keywords = ' '.join(tokens_without_sw)
-    return new_question_keywords, question
-
-
-def create_final_prompt(inputs: Dict[str, str], vectorstore, instruction_prompt, content_prompt):
-        
-        question =  inputs["question"]
-        chat_history = inputs["chat_history"]
-        
-        new_question_keywords, question = remove_q_stopwords(question)
-
-        new_question, new_question_kworded = adapt_q_from_chat_history(keyword_model, new_question_keywords, question, chat_history)
-        
-
-        print("The question passed to the vector search is:")
-        print(new_question_kworded)
-        
-        docs_keep_as_doc, docs_content, docs_url = find_relevant_passages(new_question_kworded, embeddings, k_val = 3, out_passages = 2, vec_score_cut_off = 1.3, vec_weight = 1, tfidf_weight = 0.5, svm_weight = 1)
-
-        if docs_keep_as_doc == []:
-            {"answer": "I'm sorry, I couldn't find a relevant answer to this question.", "sources":"I'm sorry, I couldn't find a relevant source for this question."}
-        
-        #new_inputs = inputs.copy()
-        #new_inputs["question"] = new_question
-        #new_inputs["chat_history"] = chat_history_str
-        
-        string_docs_content = '\n\n\n'.join(docs_content)
-        
-        #print("The draft instruction prompt is:")
-        #print(instruction_prompt)
-        
-        instruction_prompt_out = instruction_prompt.format(question=new_question, summaries=string_docs_content)
-        #print("The final instruction prompt:")
-        #print(instruction_prompt_out)
-        
-                
-        return instruction_prompt_out, string_docs_content
-
-
-# +
-def create_prompt_templates():    
-  
-    #EXAMPLE_PROMPT = PromptTemplate(
-    #    template="\nCONTENT:\n\n{page_content}\n\nSOURCE: {source}\n\n",
-    #    input_variables=["page_content", "source"],
-    #)
-
-    CONTENT_PROMPT = PromptTemplate(
-        template="{page_content}\n\n",#\n\nSOURCE: {source}\n\n",
-        input_variables=["page_content"]
-    )
-
-
-# The main prompt:
-
-    #main_prompt_template = """
-    #Answer the question using the CONTENT below:  
-
-    #CONTENT: {summaries}
-    
-    #QUESTION: {question}
-
-    #ANSWER: """
-
-    instruction_prompt_template = """
-    {summaries}
-    
-    QUESTION: {question}
-    
-    Quote relevant text above."""
-
-   
-    INSTRUCTION_PROMPT=PromptTemplate(template=instruction_prompt_template, input_variables=['question', 'summaries'])
-    
-    return INSTRUCTION_PROMPT, CONTENT_PROMPT
-
-
-# -
-
-def get_history_sources_final_input_prompt(user_input, history):
-    
-    #if chain_agent is None:
-    #    history.append((user_input, "Please click the button to submit the Huggingface API key before using the chatbot (top right)"))
-    #    return history, history, "", ""
-    print("\n==== date/time: " + str(datetime.datetime.now()) + " ====")
-    print("User input: " + user_input)
-    
-    history = history or []
-    
-
-    
-    # Create instruction prompt
-    instruction_prompt, content_prompt = create_prompt_templates()
-    instruction_prompt_out, string_docs_content =\
-                create_final_prompt({"question": user_input, "chat_history": history}, vectorstore,
-                                    instruction_prompt, content_prompt)
-    
-    sources_txt =  string_docs_content
-    
-    #print('sources_txt:')
-    #print(sources_txt)
-    
-    history.append(user_input)
-    
-    print("Output history is:")
-    print(history)
-
-    print("The output prompt is:")
-    print(instruction_prompt_out)
-    
-    return history, sources_txt, instruction_prompt_out
-
-
-# # Chat functions
-
-def produce_streaming_answer_chatbot(history, full_prompt): 
-    
-    print("The question is: ")
-    print(full_prompt)
-    
-    # Get the model and tokenizer, and tokenize the user text.
-    model_inputs = tokenizer(text=full_prompt, return_tensors="pt").to(torch_device)
-
-    # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
-    # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
-    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        model_inputs,
-        streamer=streamer,
-        max_new_tokens=512,
-        do_sample=True,
-        #top_p=top_p,
-        temperature=float(0.00001)#,
-        #top_k=top_k
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-
-    # Pull the generated text from the streamer, and update the model output.
-    
-    history[-1][1] = ""
-    for new_text in streamer:
-        history[-1][1] += new_text
-        yield history
-
-
-def user(user_message, history):
-        return gr.update(value="", interactive=False), history + [[user_message, None]]
-
-
-def add_inputs_answer_to_history(user_message, history):
-    #history.append((user_message, [-1]))
-    
-    print("History after appending is:")
-    print(history)
-    
-    
-    return history
-
-
-# # Vector / hybrid search
-
-def find_relevant_passages(new_question_kworded, embeddings, k_val, out_passages, vec_score_cut_off, vec_weight, tfidf_weight, svm_weight, vectorstore=vectorstore):
-
-            docs = vectorstore.similarity_search_with_score(new_question_kworded, k=k_val)
-            #docs = self.vstore.similarity_search_with_score(new_question_kworded, k=k_val)
-
-            # Keep only documents with a certain score
-            #docs_orig = [x[0] for x in docs]
-            docs_scores = [x[1] for x in docs]
-
-            # Only keep sources that are sufficiently relevant (i.e. similarity search score below threshold below)
-            score_more_limit = pd.Series(docs_scores) < vec_score_cut_off
-            docs_keep = list(compress(docs, score_more_limit))
-
-            if docs_keep == []:
-                docs_keep_as_doc = []
-                docs_content = []
-                docs_url = []
-                return docs_keep_as_doc, docs_content, docs_url
-            
-            
-
-            docs_keep_as_doc = [x[0] for x in docs_keep]
-            docs_keep_length = len(docs_keep_as_doc)
-
-            #print('docs_keep:')
-            #print(docs_keep)
-
-            vec_rank = [*range(1, docs_keep_length+1)]
-            vec_score = [(docs_keep_length/x)*vec_weight for x in vec_rank]
-
-            #print("vec_rank")
-            #print(vec_rank)
-
-            #print("vec_score")
-            #print(vec_score)
-
-        
-
-            # 2nd level check on retrieved docs with TFIDF
-            content_keep=[]
-            for item in docs_keep:
-                content_keep.append(item[0].page_content)
-
-            tfidf_retriever = TFIDFRetriever.from_texts(content_keep, k = k_val)
-            tfidf_result = tfidf_retriever.get_relevant_documents(new_question_kworded)
-
-            #print("TDIDF retriever result:")
-            #print(tfidf_result)
-
-            tfidf_rank=[]
-            tfidf_score = []
-
-            for vec_item in docs_keep:
-                x = 0
-                for tfidf_item in tfidf_result:
-                    x = x + 1
-                    if tfidf_item.page_content == vec_item[0].page_content:
-                        tfidf_rank.append(x)
-                        tfidf_score.append((docs_keep_length/x)*tfidf_weight)
-
-            #print("tfidf_rank:")
-            #print(tfidf_rank)
-            #print("tfidf_score:")
-            #print(tfidf_score)
-
-
-            # 3rd level check on retrieved docs with SVM retriever
-            svm_retriever = SVMRetriever.from_texts(content_keep, embeddings, k = k_val)
-            svm_result = svm_retriever.get_relevant_documents(new_question_kworded)
-
-            #print("SVM retriever result:")
-            #print(svm_result)
-         
-            svm_rank=[]
-            svm_score = []
-
-            for vec_item in docs_keep:
-                x = 0
-                for svm_item in svm_result:
-                    x = x + 1
-                    if svm_item.page_content == vec_item[0].page_content:
-                        svm_rank.append(x)
-                        svm_score.append((docs_keep_length/x)*svm_weight)
-
-            #print("svm_score:")
-            #print(svm_score)
-
-        
-            ## Calculate final score based on three ranking methods
-            final_score = [a  + b + c for a, b, c in zip(vec_score, tfidf_score, svm_score)]
-            final_rank = [sorted(final_score, reverse=True).index(x)+1 for x in final_score]
-
-            #print("Final score:")
-            #print(final_score)
-            #print("final rank:")
-            #print(final_rank)
-
-            best_rank_index_pos = []
-
-            for x in range(1,out_passages+1):
-                try:
-                    best_rank_index_pos.append(final_rank.index(x))
-                except IndexError: # catch the error
-                    pass
-
-            # Adjust best_rank_index_pos to 
-        
-            #print("Best rank positions in original vector search list:")
-            #print(best_rank_index_pos)
-
-            best_rank_pos_series = pd.Series(best_rank_index_pos)
-            #docs_keep_out = list(compress(docs_keep, best_rank_pos_series))
-
-            #print("docs_keep:")
-            #print(docs_keep)
-
-            docs_keep_out = [docs_keep[i] for i in best_rank_index_pos]
-        
-
-            #docs_keep = [(docs_keep[best_rank_pos])]
-            # Keep only 'best' options
-            docs_keep_as_doc = [x[0] for x in docs_keep_out]# [docs_keep_as_doc_filt[0]]#[x[0] for x in docs_keep_as_doc_filt] #docs_keep_as_doc_filt[0]#
-        
-            #print("docs_keep_out:")
-            #print(docs_keep_out)
-
-            # Extract content and metadata from 'winning' passages.
-
-            content=[]
-            meta_url=[]
-            score=[]
-
-            for item in docs_keep_out:
-                content.append(item[0].page_content)
-                meta_url.append(item[0].metadata['source'])
-                score.append(item[1])       
-
-            # Create df from 'winning' passages
-
-            doc_df = pd.DataFrame(list(zip(content, meta_url, score)),
-               columns =['page_content', 'meta_url', 'score'])#.iloc[[0, 1]]
-        
-            #print("docs_keep_as_doc: ")
-            #print(docs_keep_as_doc)
-
-            #print("doc_df")
-            #print(doc_df)
-
-            docs_content = doc_df['page_content'].astype(str)
-            docs_url = "https://" + doc_df['meta_url']
-        
-            #print("Docs meta url is: ")
-            #print(docs_meta_url)
-
-            #print("Docs content is: ")
-            #print(docs_content)
-
-            #docs_url = [d['source'] for d in docs_meta]
-            #print(docs_url)
-            
-            
-
-            return docs_keep_as_doc, docs_content, docs_url

chatfuncs/.ipynb_checkpoints/ingest-checkpoint.py

@@ -1,509 +0,0 @@
-# ---
-# jupyter:
-#   jupytext:
-#     formats: ipynb,py:light
-#     text_representation:
-#       extension: .py
-#       format_name: light
-#       format_version: '1.5'
-#       jupytext_version: 1.14.6
-#   kernelspec:
-#     display_name: Python 3 (ipykernel)
-#     language: python
-#     name: python3
-# ---
-
-# # Ingest website to FAISS
-
-# ## Install/ import stuff we need
-
-import os
-from pathlib import Path
-import re
-import requests
-import pandas as pd
-import dateutil.parser
-from typing import TypeVar, List
-
-from langchain.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings
-from langchain.vectorstores.faiss import FAISS
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.docstore.document import Document
-from langchain.document_loaders import PyPDFLoader
-
-import magic
-from bs4 import BeautifulSoup
-from docx import Document as Doc
-from pypdf import PdfReader
-from docx import Document
-
-PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
-# -
-
-split_strat = [".", "!", "?", "\n\n", "\n", ",", " ", ""]
-chunk_size = 1000
-chunk_overlap = 200
-
-## Overarching ingest function:
-
-
-def determine_file_type(file_path):
-    """
-    Determine the MIME type of the given file using the magic library.
-    
-    Parameters:
-        file_path (str): Path to the file.
-    
-    Returns:
-        str: MIME type of the file.
-    """
-    return magic.from_file(file_path, mime=True)
-
-def parse_pdf(file) -> List[str]:
-
-    """
-    Extract text from a PDF file.
-    
-    Parameters:
-        file_path (str): Path to the PDF file.
-    
-    Returns:
-        List[str]: Extracted text from the PDF.
-    """
-    
-    output = []
-    for i in range(0,len(file)):
-        print(file[i].name)
-        pdf = PdfReader(file[i].name) #[i]
-        for page in pdf.pages:
-            text = page.extract_text()
-            # Merge hyphenated words
-            text = re.sub(r"(\w+)-\n(\w+)", r"\1\2", text)
-            # Fix newlines in the middle of sentences
-            text = re.sub(r"(?<!\n\s)\n(?!\s\n)", " ", text.strip())
-            # Remove multiple newlines
-            text = re.sub(r"\n\s*\n", "\n\n", text)
-            output.append(text)
-    return output
-
-
-def parse_docx(file_path):
-    """
-    Reads the content of a .docx file and returns it as a string.
-
-    Parameters:
-    - file_path (str): Path to the .docx file.
-
-    Returns:
-    - str: Content of the .docx file.
-    """
-    doc = Doc(file_path)
-    full_text = []
-    for para in doc.paragraphs:
-        full_text.append(para.text)
-    return '\n'.join(full_text)
-
-
-def parse_txt(file_path):
-    """
-    Read text from a TXT or HTML file.
-    
-    Parameters:
-        file_path (str): Path to the TXT or HTML file.
-    
-    Returns:
-        str: Text content of the file.
-    """
-    with open(file_path, 'r', encoding="utf-8") as file:
-        return file.read()
-
-
-
-def parse_file(file_paths):
-    """
-    Accepts a list of file paths, determines each file's type, 
-    and passes it to the relevant parsing function.
-    
-    Parameters:
-        file_paths (list): List of file paths.
-    
-    Returns:
-        dict: A dictionary with file paths as keys and their parsed content (or error message) as values.
-    """
-    if not isinstance(file_paths, list):
-        raise ValueError("Expected a list of file paths.")
-    
-    mime_type_to_parser = {
-        'application/pdf': parse_pdf,
-        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': parse_docx,
-        'text/plain': parse_txt,
-        'text/html': parse_html
-    }
-    
-    parsed_contents = {}
-
-    for file_path in file_paths:
-        mime_type = determine_file_type(file_path)
-        if mime_type in mime_type_to_parser:
-            parsed_contents[file_path] = mime_type_to_parser[mime_type](file_path)
-        else:
-            parsed_contents[file_path] = f"Unsupported file type: {mime_type}"
-    
-    return parsed_contents
-
-
-
-
-def parse_html(page_url, div_filter="p"):
-    """
-    Determine if the source is a web URL or a local HTML file, extract the content based on the div of choice. Also tries to extract dates (WIP)
-
-    Parameters:
-        page_url (str): The web URL or local file path.
-
-    Returns:
-        str: Extracted content.
-    """
-
-    def is_web_url(s):
-    """
-    Check if the input string is a web URL.
-    """
-    return s.startswith("http://") or s.startswith("https://")
-
-    def is_local_html_file(s):
-    """
-    Check if the input string is a path to a local HTML file.
-    """
-    return (s.endswith(".html") or s.endswith(".htm")) and os.path.isfile(s)
-
-    def extract_text_from_source(source):
-    """
-    Determine if the source is a web URL or a local HTML file, 
-    and then extract its content accordingly.
-
-    Parameters:
-        source (str): The web URL or local file path.
-
-    Returns:
-        str: Extracted content.
-    """
-        if is_web_url(source):
-            response = requests.get(source)
-            response.raise_for_status()  # Raise an HTTPError for bad responses
-            return response.text
-        elif is_local_html_file(source):
-            with open(source, 'r', encoding='utf-8') as file:
-                return file.read()
-        else:
-            raise ValueError("Input is neither a valid web URL nor a local HTML file path.")
-
-    def clean_html_data(data, date_filter="", div_filt="p"):
-    """
-    Extracts and cleans data from HTML content.
-
-    Parameters:
-        data (str): HTML content to be parsed.
-        date_filter (str, optional): Date string to filter results. If set, only content with a date greater than this will be returned.
-        div_filt (str, optional): HTML tag to search for text content. Defaults to "p".
-
-    Returns:
-        tuple: Contains extracted text and date as strings. Returns empty strings if not found.
-    """
-    
-        soup = BeautifulSoup(data, 'html.parser')
-
-        # Function to exclude div with id "bar"
-        def exclude_div_with_id_bar(tag):
-            return tag.has_attr('id') and tag['id'] == 'related-links'
-
-        text_elements = soup.find_all(div_filt)
-        date_elements = soup.find_all(div_filt, {"class": "page-neutral-intro__meta"})
-    
-        # Extract date
-        date_out = ""
-        if date_elements:
-            date_out = re.search(">(.*?)<", str(date_elements[0])).group(1)
-            date_dt = dateutil.parser.parse(date_out)
-
-            if date_filter:
-                date_filter_dt = dateutil.parser.parse(date_filter)
-                if date_dt < date_filter_dt:
-                    return '', date_out
-
-        # Extract text
-        text_out_final = ""
-        if text_elements:
-            text_out_final = '\n'.join(paragraph.text for paragraph in text_elements)
-        else:
-            print(f"No elements found with tag '{div_filt}'. No text returned.")
-    
-        return text_out_final, date_out
-    
-
-    #page_url = "https://pypi.org/project/InstructorEmbedding/" #'https://www.ons.gov.uk/visualisations/censusareachanges/E09000022/index.html'
-
-    html_text = extract_text_from_source(page_url)
-    #print(page.text)
-
-    texts = []
-    metadatas = []
-
-    clean_text, date = clean_html_data(html_text, date_filter="", div_filt=div_filter)
-    texts.append(clean_text)
-    metadatas.append({"source": page_url, "date":str(date)})
-
-    return texts, metadatas
-
-
-# +
-# Convert parsed text to docs
-# -
-
-def text_to_docs(text_dict: dict, chunk_size: int = chunk_size) -> List[Document]:
-    """
-    Converts the output of parse_file (a dictionary of file paths to content)
-    to a list of Documents with metadata.
-    """
-    
-    doc_chunks = []
-
-    for file_path, content in text_dict.items():
-        ext = os.path.splitext(file_path)[1].lower()
-
-        # Depending on the file extension, handle the content
-        if ext == '.pdf':
-            docs = pdf_text_to_docs(content, chunk_size)
-        elif ext in ['.html', '.htm', '.txt', '.docx']:
-            # Assuming you want to process HTML similarly to PDF in this context
-            docs = html_text_to_docs(content, chunk_size)
-        else:
-            print(f"Unsupported file type {ext} for {file_path}. Skipping.")
-            continue
-
-        # Add filename as metadata
-        for doc in docs:
-            doc.metadata["file"] = file_path
-        
-        doc_chunks.extend(docs)
-
-    return doc_chunks
-
-
-
-def pdf_text_to_docs(text: str, chunk_size: int = chunk_size) -> List[Document]:
-    """Converts a string or list of strings to a list of Documents
-    with metadata."""
-    if isinstance(text, str):
-        # Take a single string as one page
-        text = [text]
-        
-    page_docs = [Document(page_content=page) for page in text]
-
-    # Add page numbers as metadata
-    for i, doc in enumerate(page_docs):
-        doc.metadata["page"] = i + 1
-
-    # Split pages into chunks
-    doc_chunks = []
-
-    for doc in page_docs:
-        text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=chunk_size,
-            separators=split_strat,#["\n\n", "\n", ".", "!", "?", ",", " ", ""],
-            chunk_overlap=chunk_overlap,
-        )
-        chunks = text_splitter.split_text(doc.page_content)
-
-        
-        for i, chunk in enumerate(chunks):
-            doc = Document(
-                page_content=chunk, metadata={"page": doc.metadata["page"], "chunk": i}
-            )
-            # Add sources a metadata
-            doc.metadata["page_chunk"] = f"{doc.metadata['page']}-{doc.metadata['chunk']}"
-            doc_chunks.append(doc)
-    return doc_chunks
-
-def html_text_to_docs(texts, metadatas, chunk_size:int = chunk_size):
-
-    text_splitter = RecursiveCharacterTextSplitter(
-        separators=split_strat,#["\n\n", "\n", ".", "!", "?", ",", " ", ""],
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        length_function=len
-    )
-
-    #print(texts)
-    #print(metadatas)
-
-    documents = text_splitter.create_documents(texts, metadatas=metadatas)
-
-    for i, chunk in enumerate(documents):
-        chunk.metadata["chunk"] = i + 1
-
-    return documents
-
-
-
-
-    
-
-# # Functions for working with documents after loading them back in
-
-def pull_out_data(series):
-
-    # define a lambda function to convert each string into a tuple
-    to_tuple = lambda x: eval(x)
-
-    # apply the lambda function to each element of the series
-    series_tup = series.apply(to_tuple)
-
-    series_tup_content = list(zip(*series_tup))[1]
-
-    series = pd.Series(list(series_tup_content))#.str.replace("^Main post content", "", regex=True).str.strip()
-
-    return series
-
-
-def docs_from_csv(df):
-
-    import ast
-    
-    documents = []
-    
-    page_content = pull_out_data(df["0"])
-    metadatas = pull_out_data(df["1"])
-
-    for x in range(0,len(df)):       
-        new_doc = Document(page_content=page_content[x], metadata=metadatas[x])
-        documents.append(new_doc)
-        
-    return documents
-
-
-def docs_from_lists(docs, metadatas):
-
-    documents = []
-
-    for x, doc in enumerate(docs):
-        new_doc = Document(page_content=doc, metadata=metadatas[x])
-        documents.append(new_doc)
-        
-    return documents
-
-
-def docs_elements_from_csv_save(docs_path="documents.csv"):
-
-    documents = pd.read_csv(docs_path)
-
-    docs_out = docs_from_csv(documents)
-
-    out_df = pd.DataFrame(docs_out)
-
-    docs_content = pull_out_data(out_df[0].astype(str))
-
-    docs_meta = pull_out_data(out_df[1].astype(str))
-
-    doc_sources = [d['source'] for d in docs_meta]
-
-    return out_df, docs_content, docs_meta, doc_sources
-
-
-# documents = html_text_to_docs(texts, metadatas)
-#
-# documents[0]
-#
-# pd.DataFrame(documents).to_csv("documents.csv", index=None)
-
-# ## Create embeddings and save faiss vector store to the path specified in `save_to`
-
-def load_embeddings(model_name = "hkunlp/instructor-large"):
-
-    if model_name == "hkunlp/instructor-large":
-        embeddings_func = HuggingFaceInstructEmbeddings(model_name=model_name,
-        embed_instruction="Represent the paragraph for retrieval: ",
-        query_instruction="Represent the question for retrieving supporting documents: "
-        )
-
-    else: 
-        embeddings_func = HuggingFaceEmbeddings(model_name=model_name)
-
-    global embeddings
-
-    embeddings = embeddings_func
-
-    #return embeddings_func
-
-
-def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding", model_name = "hkunlp/instructor-large"):
-
-    load_embeddings(model_name=model_name)
-
-    #embeddings_fast = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-
-    print(f"> Total split documents: {len(docs_out)}")
-
-    vectorstore = FAISS.from_documents(documents=docs_out, embedding=embeddings)
-        
-
-    if Path(save_to).exists():
-        vectorstore.save_local(folder_path=save_to)
-
-    print("> DONE")
-    print(f"> Saved to: {save_to}")
-
-    ### Save as zip, then remove faiss/pkl files to allow for upload to huggingface
-
-    import shutil
-
-    shutil.make_archive(save_to, 'zip', save_to)
-
-    os.remove(save_to + "/index.faiss")
-    os.remove(save_to + "/index.pkl")
-
-    shutil.move(save_to + '.zip', save_to + "/" + save_to + '.zip')
-
-    return vectorstore
-
-
-# +
-# https://colab.research.google.com/drive/1RWqGXd2B6sPchlYVihKaBSsHy9zWRcYF#scrollTo=Q_eTIZwf4Dk2
-
-def docs_to_chroma_save(embeddings, docs_out:PandasDataFrame, save_to:str):
-    print(f"> Total split documents: {len(docs_out)}")
-    
-    vectordb = Chroma.from_documents(documents=docs_out, 
-                                 embedding=embeddings,
-                                 persist_directory=save_to)
-    
-    # persiste the db to disk
-    vectordb.persist()
-    
-    print("> DONE")
-    print(f"> Saved to: {save_to}")
-    
-    return vectordb
-
-
-# + [markdown] jp-MarkdownHeadingCollapsed=true
-# ## Similarity search on saved vectorstore
-# -
-
-def sim_search_local_saved_vec(query, k_val, save_to="faiss_lambeth_census_embedding"):
-
-    load_embeddings()
-
-    docsearch = FAISS.load_local(folder_path=save_to, embeddings=embeddings)
-
-
-    display(Markdown(question))
-
-    search = docsearch.similarity_search_with_score(query, k=k_val)
-
-    for item in search:
-        print(item[0].page_content)
-        print(f"Page: {item[0].metadata['source']}")
-        print(f"Date: {item[0].metadata['date']}")
-        print(f"Score: {item[1]}")
-        print("---")