approach: local embeddings + anonymizing the context after indexing‚ÄãIf for some reason you would like to index the data in its original form, or simply use custom embeddings, below is an example of how to do it:anonymizer = PresidioReversibleAnonymizer(    # Faker seed is used here to make sure the same fake data is generated for the test purposes    # In production, it is recommended to remove the faker_seed parameter (it will default to None)    faker_seed=42,)anonymizer.add_recognizer(polish_id_recognizer)anonymizer.add_recognizer(time_recognizer)anonymizer.add_operators(new_operators)from langchain.embeddings import HuggingFaceBgeEmbeddingsmodel_name = "BAAI/bge-base-en-v1.5"# model_kwargs = {'device': 'cuda'}encode_kwargs = {"normalize_embeddings": True}  # set True to compute cosine similaritylocal_embeddings = HuggingFaceBgeEmbeddings(    model_name=model_name,    # model_kwargs=model_kwargs,    encode_kwargs=encode_kwargs,    query_instruction="Represent this sentence for searching relevant passages:",)text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)chunks = text_splitter.split_documents(documents)docsearch = FAISS.from_documents(chunks, local_embeddings)retriever = docsearch.as_retriever()template = """Answer the question based only on the following context:{context}Question: {anonymized_question}"""prompt = ChatPromptTemplate.from_template(template)model = ChatOpenAI(temperature=0.2)from langchain.prompts.prompt import PromptTemplatefrom langchain.schema import format_documentDEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template(template="{page_content}")def _combine_documents(    docs, document_prompt=DEFAULT_DOCUMENT_PROMPT, document_separator="\n\n"):    doc_strings = [format_document(doc, document_prompt) for doc in docs]    return document_separator.join(doc_strings)chain_with_deanonymization = (    RunnableMap({"question": RunnablePassthrough()})    | {        "context": itemgetter("question")        | retriever
Open In Colab
Open In Colab ->: approach: local embeddings + anonymizing the context after indexing‚ÄãIf for some reason you would like to index the data in its original form, or simply use custom embeddings, below is an example of how to do it:anonymizer = PresidioReversibleAnonymizer(    # Faker seed is used here to make sure the same fake data is generated for the test purposes    # In production, it is recommended to remove the faker_seed parameter (it will default to None)    faker_seed=42,)anonymizer.add_recognizer(polish_id_recognizer)anonymizer.add_recognizer(time_recognizer)anonymizer.add_operators(new_operators)from langchain.embeddings import HuggingFaceBgeEmbeddingsmodel_name = "BAAI/bge-base-en-v1.5"# model_kwargs = {'device': 'cuda'}encode_kwargs = {"normalize_embeddings": True}  # set True to compute cosine similaritylocal_embeddings = HuggingFaceBgeEmbeddings(    model_name=model_name,    # model_kwargs=model_kwargs,    encode_kwargs=encode_kwargs,    query_instruction="Represent this sentence for searching relevant passages:",)text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)chunks = text_splitter.split_documents(documents)docsearch = FAISS.from_documents(chunks, local_embeddings)retriever = docsearch.as_retriever()template = """Answer the question based only on the following context:{context}Question: {anonymized_question}"""prompt = ChatPromptTemplate.from_template(template)model = ChatOpenAI(temperature=0.2)from langchain.prompts.prompt import PromptTemplatefrom langchain.schema import format_documentDEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template(template="{page_content}")def _combine_documents(    docs, document_prompt=DEFAULT_DOCUMENT_PROMPT, document_separator="\n\n"):    doc_strings = [format_document(doc, document_prompt) for doc in docs]    return document_separator.join(doc_strings)chain_with_deanonymization = (    RunnableMap({"question": RunnablePassthrough()})    | {        "context": itemgetter("question")        | retriever