:wrench: :wrench:

.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea/
+.DS_Store
+
+# Chat data
+data/
+
+# Embedding data
+*.pkl

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Harrison Chase
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,37 @@
+# Chat with your Telegram Chat! 
+
+Understand who you are and your relationships by creating a ChatGPT like experience over your own Telegram chat with [LangChain](https://github.com/hwchase17/langchain). 
+
+Here is a very scientific peer-reviewed mathematical equation:
+
+ ```
+ Your Telegram chats ≈ Your thoughts ≈ You
+ ```
+
+
+When have you been the happiest? What triggers you instantly? How could you have been more compassionate? When do you say yes and when do you say no? At what time are you the funniest?
+
+
+Ask anything you've wanted to know about yourself and your relationship with someone.
+
+## Ingest data
+From the Telegram Desktop App, export your chat history.
+
+![](public/chat_history.png)
+
+Uncheck all the boxes and make sure you export it in `JSON`, you will then be able to download a `result.json` file.
+
+![](public/chat_history_parameters.png)
+
+You can then embed the data by running 
+
+```python ingest_data.py [result.json path]```
+
+
+## Running the Application
+
+By running `python app.py` from the command line you can easily interact with your Telegram Chat through a Gradio Web App.  
+
+
+
+*The original boilerplate is from https://github.com/hwchase17/chat-your-data*

app.py

@@ -0,0 +1,123 @@
+import os
+from typing import Optional, Tuple
+
+import gradio as gr
+import pickle
+from query_data import get_chain
+from ingest_data import embed_chat
+from threading import Lock
+
+
+def click_embed(file):
+    """Embed Telegram chat.
+       """
+    embed_chat(file.name)
+    with open("vectorstore.pkl", "rb") as f:
+        vectorstore = pickle.load(f)
+    chain = get_chain(vectorstore)
+    return chain, gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(
+        visible=True), gr.update(visible=True), gr.update(
+        visible=True), gr.update(visible=True)
+
+
+def set_openai_api_key(api_key: str):
+    """Set the api key.
+    """
+    if api_key:
+        os.environ["OPENAI_API_KEY"] = api_key
+
+
+def upload_file(file_obj):
+    return file_obj
+
+
+class ChatWrapper:
+
+    def __init__(self):
+        self.lock = Lock()
+
+    def __call__(
+            self, api_key: str, inp: str, history: Optional[Tuple[str, str]], chain
+    ):
+        """Execute the chat functionality."""
+        self.lock.acquire()
+        try:
+            history = history or []
+            # If chain is None, that is because no API key was provided.
+            if chain is None:
+                history.append((inp, "Please paste your OpenAI key to use"))
+                return history, history
+            # Set OpenAI key
+            import openai
+            openai.api_key = api_key
+            # Run chain and append input.
+            output = chain({"question": inp, "chat_history": history})["answer"]
+            history.append((inp, output))
+        except Exception as e:
+            raise e
+        finally:
+            self.lock.release()
+        return history, history
+
+
+chat = ChatWrapper()
+
+block = gr.Blocks(css=".gradio-container {background-color: lightgray}")
+
+with block:
+    with gr.Row():
+        gr.Markdown("<h3><center>Telegram Chat Chat</center></h3>")
+
+        openai_api_key_textbox = gr.Textbox(
+            placeholder="Paste your OpenAI API key (sk-...)",
+            show_label=False,
+            lines=1,
+            type="password",
+            visible=True
+        )
+        telegram_chat_file = gr.File(file_count="single", file_types=["json"], interactive=True, show_label=True,
+                                     visible=True,
+                                     label="Telegram chat history exported .json file")
+        embed_button = gr.Button("Create embeddings", visible=True)
+
+    chatbot = gr.Chatbot(visible=False)
+
+    with gr.Row():
+        message = gr.Textbox(
+            label="What's your question?",
+            placeholder="Ask questions about your Telegram conversation",
+            lines=1,
+            visible=False
+        )
+        submit = gr.Button(value="Send", variant="secondary", visible=False).style(full_width=False)
+    with gr.Row(visible=False) as examples_row:
+        gr.Examples(
+            examples=[
+                "When was [your name] happy and how does it show?",
+                "How could [your name] have been more compassionate?",
+                "What triggers [your name] instantly?",
+            ],
+            inputs=message,
+        )
+
+    gr.HTML(
+        "<center>Powered by <a href='https://github.com/hwchase17/langchain'>LangChain 🦜️🔗</a></center>"
+    )
+
+    state = gr.State()
+    agent_state = gr.State()
+
+    embed_button.click(click_embed, inputs=[telegram_chat_file],
+                       outputs=[agent_state, openai_api_key_textbox, telegram_chat_file, embed_button, chatbot, message,
+                                submit, examples_row])
+
+    submit.click(chat, inputs=[openai_api_key_textbox, message, state, agent_state], outputs=[chatbot, state])
+    print(agent_state)
+    message.submit(chat, inputs=[openai_api_key_textbox, message, state, agent_state], outputs=[chatbot, state])
+
+    openai_api_key_textbox.change(
+        set_openai_api_key,
+        inputs=[openai_api_key_textbox],
+    )
+
+block.launch(debug=True)

cli_app.py

@@ -0,0 +1,17 @@
+import pickle
+from query_data import get_chain
+
+
+if __name__ == "__main__":
+    with open("vectorstore.pkl", "rb") as f:
+        vectorstore = pickle.load(f)
+    qa_chain = get_chain(vectorstore)
+    chat_history = []
+    print("Chat with your docs!")
+    while True:
+        print("Human:")
+        question = input()
+        result = qa_chain({"question": question, "chat_history": chat_history})
+        chat_history.append((question, result["answer"]))
+        print("AI:")
+        print(result["answer"])

ingest_data.py

@@ -0,0 +1,33 @@
+import argparse
+import pickle
+
+from langchain.text_splitter import CharacterTextSplitter
+from telegram_chat_loader import TelegramChatLoader
+from langchain.vectorstores.faiss import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+
+
+# Load Data
+def embed_chat(chat_file_path):
+    loader = TelegramChatLoader(chat_file_path)
+    raw_documents = loader.load()
+
+    # Split text
+    text_splitter = CharacterTextSplitter(separator="\n\n", chunk_size=512, chunk_overlap=20)
+    documents = text_splitter.split_documents(raw_documents)
+
+    # Load Data to vectorstore
+    embeddings = OpenAIEmbeddings()
+    vectorstore = FAISS.from_documents(documents, embeddings)
+
+    # Save vectorstore
+    with open("vectorstore.pkl", "wb") as f:
+        pickle.dump(vectorstore, f)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('file_name', type=str, help='The Telegram chat exported *.json file')
+    args = parser.parse_args()
+    file_name = args.file_name
+    embed_chat(file_name)

query_data.py

@@ -0,0 +1,34 @@
+from langchain.prompts.prompt import PromptTemplate
+from langchain.llms import OpenAI
+from langchain.chains import ChatVectorDBChain
+
+_template = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question.
+You can assume the question about the conversation containing all the messages exchanged between these people.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone question:"""
+CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(_template)
+
+template = """You are an AI assistant for answering questions about this online conversation between these people.
+You are given the following extracted parts of a long document and a question. 
+Provide a conversational answer that solely comes from this online conversation between these people and your interpretation.
+Your responses should be informative, interesting, and engaging. You should respond thoroughly. 
+Question: {question}
+=========
+{context}
+=========
+Answer:"""
+QA_PROMPT = PromptTemplate(template=template, input_variables=["question", "context"])
+
+
+def get_chain(vectorstore):
+    llm = OpenAI(temperature=0)
+    qa_chain = ChatVectorDBChain.from_llm(
+        llm,
+        vectorstore,
+        qa_prompt=QA_PROMPT,
+        condense_question_prompt=CONDENSE_QUESTION_PROMPT,
+    )
+    return qa_chain

requirements.txt

@@ -0,0 +1,5 @@
+langchain
+openai
+unstructured
+faiss-cpu
+gradio

telegram_chat_loader.py

@@ -0,0 +1,47 @@
+"""Loader that loads Telegram chat json dump."""
+import json
+import pandas as pd
+from pathlib import Path
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+def concatenate_rows(row):
+    date = row['date']
+    sender = row['from']
+    text = row['text']
+    return f'{sender} on {date}: {text}\n\n'
+
+
+class TelegramChatLoader(BaseLoader):
+    """Loader that loads Telegram chat json directory dump."""
+
+    def __init__(self, path: str):
+        """Initialize with path."""
+        self.file_path = path
+
+    def load(self) -> List[Document]:
+        """Load documents."""
+        p = Path(self.file_path)
+
+        with open(p, encoding="utf8") as f:
+            d = json.load(f)
+
+        normalized_messages = pd.json_normalize(d['messages'])
+        df_normalized_messages = pd.DataFrame(normalized_messages)
+
+        # Only keep plain text messages (no services, nor links, hashtags, code, bold ...)
+        df_filtered = df_normalized_messages[
+            (df_normalized_messages.type == "message") &
+            (df_normalized_messages.text.apply(lambda x: type(x) == str))
+            ]
+
+        df_filtered = df_filtered[["date", "text", "from"]]
+
+        text = df_filtered.apply(concatenate_rows, axis=1).str.cat(sep='')
+
+        metadata = {"source": str(p)}
+
+        return [Document(page_content=text, metadata=metadata)]