Upload 25 files

app.py

@@ -0,0 +1,43 @@
+import gradio as gr
+
+
+from llm.qa_agent import QnAAgent
+from llm.calculator_agent import CalculatorAgent
+from llm.orchestrator import Orchestrator
+
+
+
+if __name__ == "__main__":
+    orchestrator = Orchestrator()
+    qna_agent = QnAAgent()
+    calculator_agent = CalculatorAgent()
+
+    # question = input("Question - ")
+    def get_answer(question:str) -> [str, str]:
+        
+        api_name, parameters = orchestrator.get_API_call(question)
+
+        print(f"Using the {api_name} Agent")
+        print(api_name, parameters)
+        if api_name == "QnA":
+            answer, wiki_page = qna_agent.get_answer(parameters)
+
+    # elif api_name == "calculator":
+    #     operand, op1, op2 = parameters.split(",")
+    #     answer = calculator_agent.calculate(operand, op1, op2)
+
+        print(answer)   
+        return [answer, wiki_page]
+    
+    demo = gr.Interface(
+        fn=get_answer,
+        inputs=gr.Textbox(placeholder="Enter your question...[Who won the Cricket World Cup in 2023?]")
+,
+        # outputs=[gr.Textbox(label=f'Document {i+1}') for i in range(TOP_K)],
+        outputs=[gr.Textbox(label="Answer"), gr.Textbox(label="Wikipedia Page")],
+        title="Real time Question Answering",
+    )
+
+    demo.launch()
+    
+

llm/calculator_agent.py

@@ -0,0 +1,18 @@
+class CalculatorAgent:
+    def calculate(self, operation: str, x: str, y: str) -> float:
+        operation = operation.lower().strip()
+
+        x = float(x)
+        y = float(y)
+        if operation == "add":
+            return x + y
+        elif operation == "subtract":
+            return x - y
+        elif operation == "multiply":
+            return x * y
+        elif operation == "divide":
+            if y == 0:
+                return "Cannot divide by zero"
+            return x / y
+        else:
+            return "Unknown operation"

llm/gemini_client.py

@@ -0,0 +1,51 @@
+import os
+
+import google.generativeai as genai
+
+
+class GeminiClient:
+    def __init__(self, system_message=None):
+        self._system_message = system_message
+        self._connect_client()
+
+    def _connect_client(self):
+        if not os.getenv("GOOGLE_PALM_KEY"):
+            raise Exception("Please set your Google MakerSuite API key")
+
+        api_key = os.getenv("GOOGLE_PALM_KEY")
+        genai.configure(api_key=api_key)
+
+        safety_settings = [
+            {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"},
+            {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_ONLY_HIGH"},
+            {
+                "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                "threshold": "BLOCK_ONLY_HIGH",
+            },
+            {
+                "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                "threshold": "BLOCK_ONLY_HIGH",
+            },
+        ]
+
+        defaults = {
+            "temperature": 0.7,
+            "top_k": 40,
+            "top_p": 0.95,
+            "max_output_tokens": 1024,
+        }
+
+        self._model = genai.GenerativeModel(
+            model_name="gemini-pro",
+            generation_config=defaults,
+            safety_settings=safety_settings,
+        )
+
+    def generate_text(self, prompt: str) -> str:
+        full_prompt = self._system_message + prompt
+        try:
+            response = self._model.generate_content(full_prompt).text
+        except Exception as e:
+            print(f"Error: {e}")
+            response = ""
+        return response

llm/orchestrator.py

@@ -0,0 +1,35 @@
+from llm.gemini_client import GeminiClient
+
+SYSTEM_MESSAGE = """You are an orchestrator that can knows what various tools 
+or agents can do and which is the right one to pick. Given a question, your
+job is just to pick the right agent to use and the rest will be taken care of.
+
+For now you can use a calculator agent that can help you do basic arithmetic
+calculations. You can also use a question answering agent that can answer
+questions about various topics. 
+
+The API's are:
+calculator[operand 1, operand 2, operation]
+QnA[question]
+
+Here are some examples:
+
+Example 1:
+Question: What is 2 + 2?
+Response: calculator$add, 2, 2
+
+Example 2: Who designed the Eiffel Tower? 
+Respnse: QnA$Who designed the Eiffel Tower?
+
+### Question:
+"""
+
+
+class Orchestrator:
+    def __init__(self):
+        self._client = GeminiClient(system_message=SYSTEM_MESSAGE)
+
+    def get_API_call(self, query: str) -> (str, str):
+        api_call = self._client.generate_text(query)
+        api_name, parameters = api_call.split("$")
+        return api_name, parameters

llm/qa_agent.py

@@ -0,0 +1,29 @@
+from llm.gemini_client import GeminiClient
+from llm.wiki_agent import WikiSearchAgent
+
+SYSTEM_MESSAGE = """You are a Question Answering tool that can answer various 
+trivia questions. However, you might be asked questions that is beyond your
+knowledge or recent events that you might not be trained on 
+(beyond training cutoff). So, if there is Wikipedia page entry provided,
+use that to answer the question. Just return the answer, don't make a 
+verbose response."""
+
+
+class QnAAgent:
+    def __init__(self):
+        self._client = GeminiClient(system_message=SYSTEM_MESSAGE)
+        self._wiki_tool = WikiSearchAgent()
+
+    @staticmethod
+    def _format_prompt(query: str, wiki_page: str) -> str:
+        return f"\n###Question:{query} \
+        \n###Wikipedia Page:{wiki_page}"
+
+    def get_answer(self, query: str, use_context: bool = True) -> [str, str]:
+        if use_context:
+            wiki_page = self._wiki_tool.get_wikipedia_entry(query)
+            prompt = self._format_prompt(query, wiki_page)
+        else:
+            wiki_page = ""
+            prompt = query
+        return self._client.generate_text(prompt), wiki_page

llm/utils/wiki_client.py

@@ -0,0 +1,20 @@
+import wikipediaapi
+
+
+class WikiClient:
+    def __init__(self):
+        self.wiki = wikipediaapi.Wikipedia(
+            user_agent="WikiAgent/0.0 [email protected]",
+            language="en",
+            extract_format=wikipediaapi.ExtractFormat.WIKI,
+        )
+
+    def get_pages(self, query):
+        pages = self.wiki.page(query)
+        return pages
+
+
+if __name__ == "__main__":
+    client = WikiClient()
+    pages = client.get_pages("Cricket World Cup")
+    print(pages.summary)

llm/wiki_agent.py

@@ -0,0 +1,46 @@
+from llm.gemini_client import GeminiClient
+
+from llm.utils.wiki_client import WikiClient
+
+SYSTEM_MESSAGE = """you have access to a wikipedia summarizer that can return a summary for a topic. \
+Your job is to act as a question answering tool. Whenever you are asked about a question related to knowledge, \
+instead of using your internal knowledge (which can be faulty or out of date), \
+format a Wikipedia search query string that can help answer the question. \
+
+Remember Wikipedia Entries are usually about a simple entity or event, so keep the \
+query short, and about the entity being asked about. Also, don't use your knowledge \
+to ask about the answer. Instead form queries about the entity in the question. This \
+will help you get the right wikipedia entries for questions when you dont know the answer
+
+### Example 1:
+
+Question: Who won the ICC Cricket World Cup?
+Correct Response: Cricket World Cup
+Incorrect response: Australia
+
+### Example 2:
+
+Question: Who directed the classic 30s western Stagecoach?
+Response: Stagecoach
+Incorrect response: John Ford
+
+
+Below is the question. Return the wikipedia search query you would use \n
+
+### Question:
+"""
+
+
+class WikiSearchAgent:
+    def __init__(self):
+        self._llm_client = GeminiClient(system_message=SYSTEM_MESSAGE)
+        self._wiki_client = WikiClient()
+
+    def get_wikipedia_entry(self, prompt: str) -> str:
+
+        wiki_search_query = self._llm_client.generate_text(prompt)
+        wikipedia_page = self._wiki_client.get_pages(wiki_search_query)
+        try:
+            return wikipedia_page.summary
+        except:
+            return ""

requirements.txt

@@ -0,0 +1,95 @@
+aiofiles==23.2.1
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+attrs==23.1.0
+beautifulsoup4==4.12.2
+cachetools==5.3.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+datasets==2.15.0
+dill==0.3.7
+fastapi==0.109.0
+ffmpy==0.3.1
+filelock==3.13.1
+fonttools==4.47.2
+frozenlist==1.4.1
+fsspec==2023.10.0
+google-ai-generativelanguage==0.4.0
+google-api-core==2.15.0
+google-auth==2.25.2
+google-generativeai==0.3.2
+googleapis-common-protos==1.62.0
+gradio==4.14.0
+gradio_client==0.8.0
+grpcio==1.60.0
+grpcio-status==1.60.0
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.26.0
+huggingface-hub==0.20.1
+idna==3.6
+importlib-resources==6.1.1
+install==1.3.5
+Jinja2==3.1.3
+jsonschema==4.21.0
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+matplotlib==3.8.2
+mdurl==0.1.2
+multidict==6.0.4
+multiprocess==0.70.15
+numpy==1.26.2
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.4
+pillow==10.2.0
+proto-plus==1.23.0
+protobuf==4.25.1
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.32.1
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.17.1
+rsa==4.9
+semantic-version==2.10.0
+setuptools==68.2.2
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.5
+starlette==0.35.1
+tomlkit==0.12.0
+toolz==0.12.0
+tqdm==4.66.1
+typer==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.3
+urllib3==2.1.0
+uvicorn==0.26.0
+websockets==11.0.3
+wheel==0.41.2
+wikipedia==1.4.0
+Wikipedia-API==0.6.0
+xxhash==3.4.1
+yarl==1.9.4

triviaQA.py

@@ -0,0 +1,62 @@
+import datasets
+
+from llm.qa_agent import QnAAgent
+
+validation_dataset = datasets.load_dataset(
+    "trivia_qa", "rc", split="test"
+)  # remove [:5%] to run on full validation set
+
+PUNCTUATION_SET_TO_EXCLUDE = set("".join(["‘", "’", "´", "`", ".", ",", "-", '"']))
+
+qna_agent = QnAAgent()
+
+
+def get_sub_answers(answers, begin=0, end=None):
+    return [" ".join(x.split(" ")[begin:end]) for x in answers if len(x.split(" ")) > 1]
+
+
+def expand_to_aliases(given_answers, make_sub_answers=False):
+    if make_sub_answers:
+        # if answers are longer than one word, make sure a predictions is correct if it coresponds to the complete 1: or :-1 sub word
+        # *e.g.* if the correct answer contains a prefix such as "the", or "a"
+        given_answers = (
+            given_answers
+            + get_sub_answers(given_answers, begin=1)
+            + get_sub_answers(given_answers, end=-1)
+        )
+    answers = []
+    for answer in given_answers:
+        alias = answer.replace("_", " ").lower()
+        alias = "".join(
+            c if c not in PUNCTUATION_SET_TO_EXCLUDE else " " for c in alias
+        )
+        answers.append(" ".join(alias.split()).strip())
+    return set(answers)
+
+
+def evaluate(example):
+    # get answer from QnA agent
+    answer_without_context = qna_agent.get_answer(example["question"], use_context=False)
+    answer_with_context = qna_agent.get_answer(example["question"], use_context=True)
+
+    example["output"] = answer_without_context
+    example["output_context"] = answer_with_context
+
+    example["targets"] = example["answer"]["aliases"]
+    answers = expand_to_aliases(example["targets"], make_sub_answers=True)
+
+    predictions = expand_to_aliases([example["output"]])
+    preditions_with_context = expand_to_aliases([example["output_context"]])
+
+    # if there is a common element, it's a match
+    example["match"] = len(list(answers & predictions)) > 0
+    example["match_context"] = len(list(answers & preditions_with_context)) > 0
+
+    return example
+
+
+results = validation_dataset.map(evaluate)
+
+print("Exact Match (EM) without context: {:.2f}".format(100 * sum(results['match'])/len(results)))
+print("Exact Match (EM) with context: {:.2f}".format(100 * sum(results['match_context'])/len(results)))
+