feat: Implement a Qdrant vector store using GTE-small embeddings. Initialized it with golf shot data collected over time.

.gitignore

@@ -204,5 +204,8 @@ dist-ssr
 *.sln
 *.sw?
 
-# Temporary directory exclusion
-frontend_backup/
+# Data files
+data/raw/*
+data/processed/*
+!data/raw/.gitkeep
+!data/processed/.gitkeep

notebooks/01_Embed_and_Store_Shots_GTE_with_Qdrant.ipynb

@@ -0,0 +1,345 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e20d8915",
+   "metadata": {},
+   "source": [
+    "# 📌 Notebook 1: Embed and Store Shot Data in Qdrant\n",
+    "\n",
+    "This notebook loads your cleaned shot data, embeds it using `bge-small-en-v1.5`, and stores the embeddings in Qdrant for use in retrieval and recommendation."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b11634a3",
+   "metadata": {},
+   "source": [
+    "# Initial Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "edc009c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 1: Initial Setup:\n",
+    "\n",
+    "# Load sentence-transformers and GTE-small model\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "model = SentenceTransformer('thenlper/gte-small')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8d5248e0",
+   "metadata": {},
+   "source": [
+    "## Load Shot Data From .csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "dce5090d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Date</th>\n",
+       "      <th>Club Type</th>\n",
+       "      <th>Club Description</th>\n",
+       "      <th>Carry Distance</th>\n",
+       "      <th>Total Distance</th>\n",
+       "      <th>Ball Speed</th>\n",
+       "      <th>Club Speed</th>\n",
+       "      <th>Spin Rate</th>\n",
+       "      <th>Attack Angle</th>\n",
+       "      <th>Descent Angle</th>\n",
+       "      <th>Shot Classification</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2025-02-04 12:41:00</td>\n",
+       "      <td>Driver</td>\n",
+       "      <td>TopGolf - Driver (+1; N; 2.75T)</td>\n",
+       "      <td>124.33</td>\n",
+       "      <td>171.19</td>\n",
+       "      <td>122.16</td>\n",
+       "      <td>85.92</td>\n",
+       "      <td>1154</td>\n",
+       "      <td>2.95</td>\n",
+       "      <td>11.33</td>\n",
+       "      <td>Hook</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2025-02-04 12:41:42</td>\n",
+       "      <td>Driver</td>\n",
+       "      <td>TopGolf - Driver (+1; N; 2.75T)</td>\n",
+       "      <td>104.75</td>\n",
+       "      <td>150.95</td>\n",
+       "      <td>120.35</td>\n",
+       "      <td>84.20</td>\n",
+       "      <td>1666</td>\n",
+       "      <td>2.45</td>\n",
+       "      <td>8.19</td>\n",
+       "      <td>Push Hook</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2025-02-04 12:42:17</td>\n",
+       "      <td>Driver</td>\n",
+       "      <td>TopGolf - Driver (+1; N; 2.75T)</td>\n",
+       "      <td>163.45</td>\n",
+       "      <td>195.51</td>\n",
+       "      <td>115.05</td>\n",
+       "      <td>86.28</td>\n",
+       "      <td>1227</td>\n",
+       "      <td>4.30</td>\n",
+       "      <td>23.02</td>\n",
+       "      <td>Push</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2025-02-04 12:43:05</td>\n",
+       "      <td>Driver</td>\n",
+       "      <td>TopGolf - Driver (+1; N; 2.75T)</td>\n",
+       "      <td>162.57</td>\n",
+       "      <td>192.56</td>\n",
+       "      <td>110.91</td>\n",
+       "      <td>81.96</td>\n",
+       "      <td>1783</td>\n",
+       "      <td>1.74</td>\n",
+       "      <td>24.87</td>\n",
+       "      <td>Push</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2025-02-04 12:44:18</td>\n",
+       "      <td>Driver</td>\n",
+       "      <td>TopGolf - Driver (+1; N; 2.75T)</td>\n",
+       "      <td>105.30</td>\n",
+       "      <td>152.00</td>\n",
+       "      <td>118.83</td>\n",
+       "      <td>80.78</td>\n",
+       "      <td>1478</td>\n",
+       "      <td>1.29</td>\n",
+       "      <td>8.67</td>\n",
+       "      <td>Push Draw</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                  Date Club Type                 Club Description  \\\n",
+       "0  2025-02-04 12:41:00    Driver  TopGolf - Driver (+1; N; 2.75T)   \n",
+       "1  2025-02-04 12:41:42    Driver  TopGolf - Driver (+1; N; 2.75T)   \n",
+       "2  2025-02-04 12:42:17    Driver  TopGolf - Driver (+1; N; 2.75T)   \n",
+       "3  2025-02-04 12:43:05    Driver  TopGolf - Driver (+1; N; 2.75T)   \n",
+       "4  2025-02-04 12:44:18    Driver  TopGolf - Driver (+1; N; 2.75T)   \n",
+       "\n",
+       "   Carry Distance  Total Distance  Ball Speed  Club Speed  Spin Rate  \\\n",
+       "0          124.33          171.19      122.16       85.92       1154   \n",
+       "1          104.75          150.95      120.35       84.20       1666   \n",
+       "2          163.45          195.51      115.05       86.28       1227   \n",
+       "3          162.57          192.56      110.91       81.96       1783   \n",
+       "4          105.30          152.00      118.83       80.78       1478   \n",
+       "\n",
+       "   Attack Angle  Descent Angle Shot Classification  \n",
+       "0          2.95          11.33                Hook  \n",
+       "1          2.45           8.19           Push Hook  \n",
+       "2          4.30          23.02                Push  \n",
+       "3          1.74          24.87                Push  \n",
+       "4          1.29           8.67           Push Draw  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Step 2: Load cleaned shot data\n",
+    "import pandas as pd\n",
+    "shot_data = pd.read_csv('../data/raw/cleaned_shot_data.csv')\n",
+    "shot_data.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "56e952f9",
+   "metadata": {},
+   "source": [
+    "## Embed Shot Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "a5052716",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 3: Format shot data into text chunks for embedding\n",
+    "def create_embedding_text(row):\n",
+    "    return f\"{row['Club Type']} | Carry: {row['Carry Distance']} yds | Ball Speed: {row['Ball Speed']} mph | Classification: {row['Shot Classification']}\"\n",
+    "\n",
+    "texts = shot_data.apply(create_embedding_text, axis=1).tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "fbd10b62",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Batches: 100%|██████████| 16/16 [00:17<00:00,  1.10s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Generate embeddings for shot text\n",
+    "embeddings = model.encode(texts, show_progress_bar=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b5744da3",
+   "metadata": {},
+   "source": [
+    "## Upload Embedded Shot Data to Qdrant"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "d95b07f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Collect the Qdrant API key\n",
+    "from getpass import getpass\n",
+    "\n",
+    "qdrant_api_key = getpass('🔑 Enter your Qdrant API Key: ')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "c4a90e06",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/5p/gq47dsys3k5663k1r5z8s3c40000gn/T/ipykernel_54911/3543225837.py:11: DeprecationWarning: `recreate_collection` method is deprecated and will be removed in the future. Use `collection_exists` to check collection existence and `create_collection` instead.\n",
+      "  client.recreate_collection(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Qdrant setup\n",
+    "from qdrant_client import QdrantClient\n",
+    "from qdrant_client.models import VectorParams, PointStruct, Distance\n",
+    "\n",
+    "client = QdrantClient(\n",
+    "    url='https://6f592f43-f667-4234-ad3a-4f15ed5882ef.us-west-2-0.aws.cloud.qdrant.io:6333',\n",
+    "    api_key=qdrant_api_key\n",
+    ")\n",
+    "\n",
+    "# Recreate the collection to flush old data\n",
+    "client.recreate_collection(\n",
+    "    collection_name='golf_shot_vectors',\n",
+    "    vectors_config=VectorParams(size=384, distance=Distance.COSINE)\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "f5c0f891",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "UpdateResult(operation_id=0, status=<UpdateStatus.COMPLETED: 'completed'>)"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Upload embedded vectors to Qdrant\n",
+    "points = [\n",
+    "    PointStruct(id=i, vector=embeddings[i], payload={'text': texts[i]})\n",
+    "    for i in range(len(embeddings))\n",
+    "]\n",
+    "client.upsert(collection_name='golf_shot_vectors', points=points)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/02_Retrieve_and_Test_Recommendations_GTE_with_Qdrant.ipynb

@@ -0,0 +1,138 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f36ee4e7",
+   "metadata": {},
+   "source": [
+    " # 📌 Notebook 2: Retrieve and Test Recommendations (GTE-small)\n",
+    "This notebook allows you to enter a shot scenario and retrieve semantically similar historical shots using Qdrant with GTE-small embeddings.\n",
+    "   "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aabcef6d",
+   "metadata": {},
+   "source": [
+    "## Initial Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "b3a051bc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/mwalker/development/TAMARKDesigns/AI-Maker-Space/cohort-6/projects/session-05/AIE6-Golf-Agent/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Step 1: Initial Setup:\n",
+    "\n",
+    "# Load sentence-transformers and GTE-small model\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "model = SentenceTransformer('thenlper/gte-small')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "83fa3f1d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Embed the query using GTE-small\n",
+    "\n",
+    "def get_embedding(text):\n",
+    "    return model.encode(text)\n",
+    "\n",
+    "# Define a test query\n",
+    "query = \"What club should I use from 145 yards if I want to avoid a slice?\"\n",
+    "\n",
+    "query_vector = get_embedding(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "bfa9fd99",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Collect the Qdrant API key\n",
+    "from getpass import getpass\n",
+    "\n",
+    "qdrant_api_key = getpass('🔑 Enter your Qdrant API Key: ')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "42a05a2e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Score: 0.8943 | Text: Approach Wedge | Carry: 99.81 yds | Ball Speed: 74.13 mph | Classification: Slice\n",
+      "Score: 0.8941 | Text: Approach Wedge | Carry: 99.57 yds | Ball Speed: 74.27 mph | Classification: Slice\n",
+      "Score: 0.8869 | Text: Approach Wedge | Carry: 52.12 yds | Ball Speed: 52.12 mph | Classification: Push Slice\n",
+      "Score: 0.8868 | Text: Lob Wedge | Carry: 67.72 yds | Ball Speed: 59.75 mph | Classification: Slice\n",
+      "Score: 0.8867 | Text: Lob Wedge | Carry: 41.49 yds | Ball Speed: 45.45 mph | Classification: Slice\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Retrieve top 5 most similar entries\n",
+    "from qdrant_client import QdrantClient\n",
+    "\n",
+    "client = QdrantClient(\n",
+    "    url=\"https://6f592f43-f667-4234-ad3a-4f15ed5882ef.us-west-2-0.aws.cloud.qdrant.io:6333\",\n",
+    "    api_key=qdrant_api_key\n",
+    ")\n",
+    "\n",
+    "collection_name = \"golf_shot_vectors\"\n",
+    "\n",
+    "search_result = client.query_points(\n",
+    "    collection_name=collection_name,\n",
+    "    query=query_vector,  # Pass the vector directly\n",
+    "    limit=5,\n",
+    "    with_payload=True\n",
+    "    )\n",
+    "    \n",
+    "# Access the points from the QueryResponse\n",
+    "for point in search_result.points:\n",
+    "    print(f\"Score: {point.score:.4f} | Text: {point.payload['text']}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -12,13 +12,17 @@ dependencies = [
     "pypdf",  # for PDF processing
     "PyPDF2>=3.0.0",
     "numpy>=1.24.0",
+    "pandas>=2.0.0",  # for data analysis
     "langchain>=0.1.0",
     "langchain-community>=0.0.22",
-    "langchain-openai>=0.0.8",
+    "langchain-openai>=0.0.8", 
+    "qdrant-client>=1.14.2",  # Vector storage
     "pytest>=7.0.0",  # Added pytest dependency
-    "pytest-asyncio>=0.23.0",  # Added pytest-asyncio dependency
+    "pytest-asyncio>=0.23.0",  # Added pytest-asyncio dependency    
+    "pytest-cov>=4.1.0",  # Coverage reporting
     "tavily-python>=0.3.0",  # Added tavily dependency
     "langgraph>=0.0.15",  # Added langgraph dependency
+    "sentence-transformers", #used for embeddings 
 ]
 
 [build-system]