Spaces:
Sleeping
Sleeping
Carlos Isael Ramírez González
commited on
Commit
·
56ff037
1
Parent(s):
a3373fd
Modelo nuevo completado
Browse files- app.py +15 -9
- config.py +3 -2
- examples.json +282 -0
- intelligent_question_router.py +25 -0
- load_json.py +14 -0
- memory.py +23 -44
- mojica_agent.py +234 -229
- requirements.txt +0 -0
- semantic_classifier.py +63 -0
- supervised_classifier.py +93 -0
app.py
CHANGED
@@ -9,33 +9,39 @@ app = FastAPI()
|
|
9 |
|
10 |
mojica_bot = MojicaAgent(Config)
|
11 |
|
|
|
12 |
# * Esquema de entrada como marshmellow
|
13 |
-
class QuestionRequest(BaseModel):
|
14 |
question: str
|
15 |
-
|
|
|
|
|
16 |
class AnswerResponse(BaseModel):
|
17 |
-
sql: str
|
18 |
result: Any
|
19 |
-
|
|
|
|
|
20 |
@app.post("/")
|
21 |
def ask_question(req: QuestionRequest):
|
22 |
sql, result = mojica_bot.consult(req.question)
|
23 |
|
24 |
# Si es dataframe lo convertimos a json
|
25 |
-
if isinstance(result, pd.DataFrame):
|
26 |
result = result.to_dict(orient="records")
|
27 |
|
28 |
return {"sql": sql, "result": result}
|
29 |
# return {"sql": "WASA"}
|
30 |
|
|
|
31 |
# @app.post("/", response_model=AnswerResponse)
|
32 |
-
# def ask_question(req: QuestionRequest):
|
33 |
# sql, result = mojica_bot.consult(req.question)
|
34 |
-
|
35 |
# # * Si es dataframe lo convertimos a json
|
36 |
-
# if isinstance(result, pd.DataFrame):
|
37 |
# result = result.to_dict(orient="records")
|
38 |
-
|
39 |
# return {"sql": sql, "result": result}
|
40 |
|
41 |
# @app.get("/")
|
|
|
9 |
|
10 |
mojica_bot = MojicaAgent(Config)
|
11 |
|
12 |
+
|
13 |
# * Esquema de entrada como marshmellow
|
14 |
+
class QuestionRequest(BaseModel):
|
15 |
question: str
|
16 |
+
number: str
|
17 |
+
|
18 |
+
|
19 |
class AnswerResponse(BaseModel):
|
20 |
+
sql: str
|
21 |
result: Any
|
22 |
+
|
23 |
+
|
24 |
+
|
25 |
@app.post("/")
|
26 |
def ask_question(req: QuestionRequest):
|
27 |
sql, result = mojica_bot.consult(req.question)
|
28 |
|
29 |
# Si es dataframe lo convertimos a json
|
30 |
+
if isinstance(result, pd.DataFrame):
|
31 |
result = result.to_dict(orient="records")
|
32 |
|
33 |
return {"sql": sql, "result": result}
|
34 |
# return {"sql": "WASA"}
|
35 |
|
36 |
+
|
37 |
# @app.post("/", response_model=AnswerResponse)
|
38 |
+
# def ask_question(req: QuestionRequest):
|
39 |
# sql, result = mojica_bot.consult(req.question)
|
40 |
+
|
41 |
# # * Si es dataframe lo convertimos a json
|
42 |
+
# if isinstance(result, pd.DataFrame):
|
43 |
# result = result.to_dict(orient="records")
|
44 |
+
|
45 |
# return {"sql": sql, "result": result}
|
46 |
|
47 |
# @app.get("/")
|
config.py
CHANGED
@@ -5,11 +5,12 @@ from huggingface_hub import hf_hub_download
|
|
5 |
class Config:
|
6 |
DB_PATH = '/tmp/dataset.db'
|
7 |
TABLE_NAME = 'mojica_Ventas'
|
|
|
8 |
MODEL_NAME = "ibm-granite/granite-3b-code-instruct-128k"
|
9 |
CSV_PATH = "/kaggle/input/mojica-hoja-1/mojica_hoja_1.csv"
|
10 |
MAX_HISTORY = 3 # Mantener las últimas 3 interacciones (memoria)
|
11 |
-
MAX_TOKENS =
|
12 |
-
MAX_NEW_TOKENS =
|
13 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
14 |
|
15 |
if not os.path.exists(Config.DB_PATH):
|
|
|
5 |
class Config:
|
6 |
DB_PATH = '/tmp/dataset.db'
|
7 |
TABLE_NAME = 'mojica_Ventas'
|
8 |
+
EXMAPLES_JSON = '/examples.json'
|
9 |
MODEL_NAME = "ibm-granite/granite-3b-code-instruct-128k"
|
10 |
CSV_PATH = "/kaggle/input/mojica-hoja-1/mojica_hoja_1.csv"
|
11 |
MAX_HISTORY = 3 # Mantener las últimas 3 interacciones (memoria)
|
12 |
+
MAX_TOKENS = 4096
|
13 |
+
MAX_NEW_TOKENS = 256
|
14 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
15 |
|
16 |
if not os.path.exists(Config.DB_PATH):
|
examples.json
ADDED
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"PRODUCTOS": [
|
3 |
+
{
|
4 |
+
"pregunta": "Top 5 productos más vendidos este año",
|
5 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND strftime('%Y', \"Fecha\") = '2025'\nGROUP BY \"Descripcion\"\nORDER BY total_vendido DESC\nLIMIT 5;"
|
6 |
+
},
|
7 |
+
{
|
8 |
+
"pregunta": "Productos con mayor margen de ganancia",
|
9 |
+
"query": "SELECT \"Descripcion\", (SUM(\"Neto\") / SUM(\"Cantidad\")) AS margen_unitario\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Cantidad\" > 0\nGROUP BY \"Descripcion\"\nHAVING SUM(\"Cantidad\") > 30\nORDER BY margen_unitario DESC\nLIMIT 10;"
|
10 |
+
},
|
11 |
+
{
|
12 |
+
"pregunta": "Productos con menor rotación en inventario",
|
13 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Fecha\" BETWEEN DATE('now','-6 months') AND DATE('now')\nGROUP BY \"Descripcion\"\nORDER BY total_vendido ASC\nLIMIT 5;"
|
14 |
+
}
|
15 |
+
],
|
16 |
+
"CLIENTES_CERO": [
|
17 |
+
{
|
18 |
+
"pregunta": "Total de clientes cero actuales",
|
19 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n);"
|
20 |
+
},
|
21 |
+
{
|
22 |
+
"pregunta": "Clientes cero que fueron VIP anteriormente",
|
23 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS historial_compra\nFROM \"sells\"\nWHERE \"Cliente\" IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n )\n) AND \"Cliente\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING SUM(\"Neto\") > 5000\nORDER BY historial_compra DESC;"
|
24 |
+
},
|
25 |
+
{
|
26 |
+
"pregunta": "Clientes cero por antigüedad de inactividad",
|
27 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", MAX(DATE(\"Fecha\")) AS ultima_compra,\n(julianday('now') - julianday(MAX(\"Fecha\"))) AS dias_inactivo\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n)\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY dias_inactivo DESC\nLIMIT 10;"
|
28 |
+
}
|
29 |
+
],
|
30 |
+
"CIUDAD_CERO": [
|
31 |
+
{
|
32 |
+
"pregunta": "Clientes cero por ciudad",
|
33 |
+
"query": "SELECT \"Ciudad\", COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n)\nGROUP BY \"Ciudad\"\nORDER BY clientes_cero DESC;"
|
34 |
+
},
|
35 |
+
{
|
36 |
+
"pregunta": "Ciudad con mayor porcentaje de clientes cero",
|
37 |
+
"query": "SELECT \"Ciudad\", \n(COUNT(DISTINCT CASE WHEN \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n) THEN \"Cliente\" END) * 100.0 / COUNT(DISTINCT \"Cliente\")) AS porcentaje_cero\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY porcentaje_cero DESC\nLIMIT 5;"
|
38 |
+
},
|
39 |
+
{
|
40 |
+
"pregunta": "Clientes cero en Guadalajara con historial de compras",
|
41 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS historial_compra\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n)\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING SUM(\"Neto\") > 1000\nORDER BY historial_compra DESC;"
|
42 |
+
}
|
43 |
+
],
|
44 |
+
"PRODUCTO_CIUDAD": [
|
45 |
+
{
|
46 |
+
"pregunta": "Producto más vendido por ciudad",
|
47 |
+
"query": "SELECT \"Ciudad\", \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL AND \"Descripcion\" IS NOT NULL\nGROUP BY \"Ciudad\", \"Descripcion\"\nORDER BY \"Ciudad\", total_vendido DESC;"
|
48 |
+
},
|
49 |
+
{
|
50 |
+
"pregunta": "Ciudades donde el queso gouda es más popular",
|
51 |
+
"query": "SELECT \"Ciudad\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Descripcion\" LIKE '%Queso Gouda%' AND \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY total_vendido DESC\nLIMIT 5;"
|
52 |
+
},
|
53 |
+
{
|
54 |
+
"pregunta": "Productos exclusivos por ciudad",
|
55 |
+
"query": "SELECT \"Ciudad\", \"Descripcion\"\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\", \"Descripcion\"\nHAVING COUNT(DISTINCT \"Ciudad\") = 1\nORDER BY \"Ciudad\";"
|
56 |
+
}
|
57 |
+
],
|
58 |
+
"CLIENTE_CIUDAD": [
|
59 |
+
{
|
60 |
+
"pregunta": "Clientes más valiosos por ciudad",
|
61 |
+
"query": "SELECT \"Ciudad\", \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS valor_total\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL AND \"Cliente\" IS NOT NULL\nGROUP BY \"Ciudad\", \"Cliente\", \"Razon Social\"\nORDER BY \"Ciudad\", valor_total DESC;"
|
62 |
+
},
|
63 |
+
{
|
64 |
+
"pregunta": "Ciudad con clientes más frecuentes",
|
65 |
+
"query": "SELECT \"Ciudad\", AVG(compras_por_cliente) AS frecuencia_promedio\nFROM (\n SELECT \"Ciudad\", \"Cliente\", COUNT(*) AS compras_por_cliente\n FROM \"sells\"\n WHERE \"Ciudad\" IS NOT NULL AND \"Cliente\" IS NOT NULL\n GROUP BY \"Ciudad\", \"Cliente\"\n)\nGROUP BY \"Ciudad\"\nORDER BY frecuencia_promedio DESC;"
|
66 |
+
},
|
67 |
+
{
|
68 |
+
"pregunta": "Distribución de clientes por ciudad",
|
69 |
+
"query": "SELECT \"Ciudad\", COUNT(DISTINCT \"Cliente\") AS total_clientes\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL AND \"Cliente\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY total_clientes DESC;"
|
70 |
+
}
|
71 |
+
],
|
72 |
+
"PRODUCTO_DINERO": [
|
73 |
+
{
|
74 |
+
"pregunta": "Productos que generan más ingresos",
|
75 |
+
"query": "SELECT \"Descripcion\", SUM(\"Neto\") AS ingresos_totales\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY \"Descripcion\"\nORDER BY ingresos_totales DESC\nLIMIT 10;"
|
76 |
+
},
|
77 |
+
{
|
78 |
+
"pregunta": "Rentabilidad por categoría de producto",
|
79 |
+
"query": "SELECT \nCASE \n WHEN \"Descripcion\" LIKE '%queso%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%pan%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%bebida%' THEN 'Bebidas'\n ELSE 'Otros'\nEND AS categoria,\nSUM(\"Neto\") AS ingresos_totales\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY categoria\nORDER BY ingresos_totales DESC;"
|
80 |
+
},
|
81 |
+
{
|
82 |
+
"pregunta": "Productos con mejor margen por unidad",
|
83 |
+
"query": "SELECT \"Descripcion\", (SUM(\"Neto\") / SUM(\"Cantidad\")) AS margen_unitario\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Cantidad\" > 0\nGROUP BY \"Descripcion\"\nHAVING SUM(\"Cantidad\") > 20\nORDER BY margen_unitario DESC\nLIMIT 10;"
|
84 |
+
}
|
85 |
+
],
|
86 |
+
"CIUDAD_DINERO": [
|
87 |
+
{
|
88 |
+
"pregunta": "Ciudades con mayor facturación",
|
89 |
+
"query": "SELECT \"Ciudad\", SUM(\"Neto\") AS ingresos_totales\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY ingresos_totales DESC\nLIMIT 5;"
|
90 |
+
},
|
91 |
+
{
|
92 |
+
"pregunta": "Crecimiento de ventas por ciudad",
|
93 |
+
"query": "SELECT \"Ciudad\", \n(SUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = '2025-06' THEN \"Neto\" ELSE 0 END) - \nSUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = '2025-05' THEN \"Neto\" ELSE 0 END)) AS crecimiento_mensual\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY crecimiento_mensual DESC;"
|
94 |
+
},
|
95 |
+
{
|
96 |
+
"pregunta": "Ticket promedio por ciudad",
|
97 |
+
"query": "SELECT \"Ciudad\", AVG(\"Neto\") AS ticket_promedio\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY ticket_promedio DESC;"
|
98 |
+
}
|
99 |
+
],
|
100 |
+
"CLIENTE_DINERO": [
|
101 |
+
{
|
102 |
+
"pregunta": "Top 10 clientes por valor de compras",
|
103 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS valor_total\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY valor_total DESC\nLIMIT 10;"
|
104 |
+
},
|
105 |
+
{
|
106 |
+
"pregunta": "Clientes con mayor frecuencia de compra de alto valor",
|
107 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", COUNT(*) AS compras, AVG(\"Neto\") AS ticket_promedio\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL AND \"Neto\" > 500\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY compras DESC\nLIMIT 10;"
|
108 |
+
},
|
109 |
+
{
|
110 |
+
"pregunta": "Clientes con mayor potencial de crecimiento",
|
111 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", \n(MAX(\"Neto\") - AVG(\"Neto\")) AS potencial_crecimiento\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING COUNT(*) > 3\nORDER BY potencial_crecimiento DESC\nLIMIT 10;"
|
112 |
+
}
|
113 |
+
],
|
114 |
+
"TIEMPO_DINERO": [
|
115 |
+
{
|
116 |
+
"pregunta": "Ventas mensuales del año actual",
|
117 |
+
"query": "SELECT strftime('%Y-%m', \"Fecha\") AS mes, SUM(\"Neto\") AS ventas_mensuales\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL AND strftime('%Y', \"Fecha\") = '2025'\nGROUP BY mes\nORDER BY mes;"
|
118 |
+
},
|
119 |
+
{
|
120 |
+
"pregunta": "Crecimiento interanual de ventas",
|
121 |
+
"query": "SELECT \nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2025' THEN \"Neto\" ELSE 0 END) AS ventas_2025,\nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2024' THEN \"Neto\" ELSE 0 END) AS ventas_2024,\n((SUM(CASE WHEN strftime('%Y', \"Fecha\") = '2025' THEN \"Neto\" ELSE 0 END) - \nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2024' THEN \"Neto\" ELSE 0 END)) * 100.0 / \nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2024' THEN \"Neto\" ELSE 0 END)) AS crecimiento_porcentual\nFROM \"sells\";"
|
122 |
+
},
|
123 |
+
{
|
124 |
+
"pregunta": "Ventas por día de la semana",
|
125 |
+
"query": "SELECT \nCASE strftime('%w', \"Fecha\")\n WHEN '0' THEN 'Domingo'\n WHEN '1' THEN 'Lunes'\n WHEN '2' THEN 'Martes'\n WHEN '3' THEN 'Miércoles'\n WHEN '4' THEN 'Jueves'\n WHEN '5' THEN 'Viernes'\n WHEN '6' THEN 'Sábado'\nEND AS dia_semana,\nSUM(\"Neto\") AS ventas_dia\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL\nGROUP BY dia_semana\nORDER BY ventas_dia DESC;"
|
126 |
+
}
|
127 |
+
],
|
128 |
+
"CIUDADES": [
|
129 |
+
{
|
130 |
+
"pregunta": "¿Cuántas ciudades diferentes tenemos registradas?",
|
131 |
+
"query": "SELECT COUNT(DISTINCT \"Ciudad\") AS total_ciudades\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL;"
|
132 |
+
},
|
133 |
+
{
|
134 |
+
"pregunta": "Lista de las 10 ciudades con más transacciones",
|
135 |
+
"query": "SELECT \"Ciudad\", COUNT(*) AS total_transacciones\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY total_transacciones DESC\nLIMIT 10;"
|
136 |
+
},
|
137 |
+
{
|
138 |
+
"pregunta": "Nombra 15 ciudades al azar de nuestra base",
|
139 |
+
"query": "SELECT DISTINCT \"Ciudad\"\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nORDER BY RANDOM()\nLIMIT 15;"
|
140 |
+
}
|
141 |
+
],
|
142 |
+
"CLIENTES": [
|
143 |
+
{
|
144 |
+
"pregunta": "¿Cuántos clientes únicos tenemos registrados?",
|
145 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS total_clientes\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL;"
|
146 |
+
},
|
147 |
+
{
|
148 |
+
"pregunta": "Muestra 10 clientes al azar con su razón social",
|
149 |
+
"query": "SELECT DISTINCT \"Cliente\", \"Razon Social\"\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nORDER BY RANDOM()\nLIMIT 10;"
|
150 |
+
},
|
151 |
+
{
|
152 |
+
"pregunta": "Clientes con nombres más largos en la base",
|
153 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", LENGTH(\"Razon Social\") AS longitud_nombre\nFROM \"sells\"\nWHERE \"Razon Social\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY longitud_nombre DESC\nLIMIT 10;"
|
154 |
+
}
|
155 |
+
],
|
156 |
+
"FECHAS": [
|
157 |
+
{
|
158 |
+
"pregunta": "¿Desde cuándo tenemos registros de ventas?",
|
159 |
+
"query": "SELECT MIN(DATE(\"Fecha\")) AS primera_fecha, MAX(DATE(\"Fecha\")) AS ultima_fecha\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL;"
|
160 |
+
},
|
161 |
+
{
|
162 |
+
"pregunta": "¿Cuántos días diferentes con ventas tenemos?",
|
163 |
+
"query": "SELECT COUNT(DISTINCT DATE(\"Fecha\")) AS dias_con_ventas\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL;"
|
164 |
+
},
|
165 |
+
{
|
166 |
+
"pregunta": "Rango de fechas de nuestro historial",
|
167 |
+
"query": "SELECT MIN(\"Fecha\") AS inicio_historial, MAX(\"Fecha\") AS fin_historial,\nJULIANDAY(MAX(\"Fecha\")) - JULIANDAY(MIN(\"Fecha\")) AS dias_totales\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL;"
|
168 |
+
}
|
169 |
+
],
|
170 |
+
"VOLUMEN_VENTAS": [
|
171 |
+
{
|
172 |
+
"pregunta": "¿Cuántas transacciones totales tenemos?",
|
173 |
+
"query": "SELECT COUNT(*) AS total_transacciones\nFROM \"sells\";"
|
174 |
+
},
|
175 |
+
{
|
176 |
+
"pregunta": "Promedio de ventas por día",
|
177 |
+
"query": "SELECT AVG(ventas_dia) AS promedio_ventas_diarias\nFROM (\n SELECT DATE(\"Fecha\"), SUM(\"Neto\") AS ventas_dia\n FROM \"sells\"\n WHERE \"Fecha\" IS NOT NULL\n GROUP BY DATE(\"Fecha\")\n);"
|
178 |
+
},
|
179 |
+
{
|
180 |
+
"pregunta": "Día con mayor número de transacciones",
|
181 |
+
"query": "SELECT DATE(\"Fecha\") AS dia, COUNT(*) AS total_transacciones\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL\nGROUP BY dia\nORDER BY total_transacciones DESC\nLIMIT 1;"
|
182 |
+
}
|
183 |
+
],
|
184 |
+
"ESTADISTICAS_BASICAS": [
|
185 |
+
{
|
186 |
+
"pregunta": "Valor promedio de una venta",
|
187 |
+
"query": "SELECT AVG(\"Neto\") AS valor_promedio_venta\nFROM \"sells\"\nWHERE \"Neto\" IS NOT NULL;"
|
188 |
+
},
|
189 |
+
{
|
190 |
+
"pregunta": "Cantidad promedio de productos por venta",
|
191 |
+
"query": "SELECT AVG(\"Cantidad\") AS cantidad_promedio\nFROM \"sells\"\nWHERE \"Cantidad\" IS NOT NULL;"
|
192 |
+
},
|
193 |
+
{
|
194 |
+
"pregunta": "Valor máximo y mínimo de una venta",
|
195 |
+
"query": "SELECT MAX(\"Neto\") AS venta_maxima, MIN(\"Neto\") AS venta_minima\nFROM \"sells\"\nWHERE \"Neto\" IS NOT NULL;"
|
196 |
+
}
|
197 |
+
],
|
198 |
+
"CATEGORIAS_PRODUCTOS": [
|
199 |
+
{
|
200 |
+
"pregunta": "¿Cuáles categorías de productos tenemos?",
|
201 |
+
"query": "SELECT DISTINCT \nCASE \n WHEN \"Descripcion\" LIKE '%QUESO%' OR \"Descripcion\" LIKE '%QUESO%' THEN 'Quesos'\n WHEN \"Descripcion\" LIKE '%LECHE%' OR \"Descripcion\" LIKE '%LACTEO%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%PAN%' OR \"Descripcion\" LIKE '%BOLLERIA%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%BEBIDA%' OR \"Descripcion\" LIKE '%REFRESCO%' THEN 'Bebidas'\n WHEN \"Descripcion\" LIKE '%EMBUTIDO%' OR \"Descripcion\" LIKE '%SALCHICHA%' THEN 'Embutidos'\n ELSE 'Otros'\nEND AS categoria\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nORDER BY categoria;"
|
202 |
+
},
|
203 |
+
{
|
204 |
+
"pregunta": "Cantidad de productos por categoría",
|
205 |
+
"query": "SELECT \nCASE \n WHEN \"Descripcion\" LIKE '%QUESO%' OR \"Descripcion\" LIKE '%QUESO%' THEN 'Quesos'\n WHEN \"Descripcion\" LIKE '%LECHE%' OR \"Descripcion\" LIKE '%LACTEO%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%PAN%' OR \"Descripcion\" LIKE '%BOLLERIA%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%BEBIDA%' OR \"Descripcion\" LIKE '%REFRESCO%' THEN 'Bebidas'\n WHEN \"Descripcion\" LIKE '%EMBUTIDO%' OR \"Descripcion\" LIKE '%SALCHICHA%' THEN 'Embutidos'\n ELSE 'Otros'\nEND AS categoria,\nCOUNT(DISTINCT \"Descripcion\") AS cantidad_productos\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY categoria\nORDER BY cantidad_productos DESC;"
|
206 |
+
},
|
207 |
+
{
|
208 |
+
"pregunta": "Categorías con mayor variedad de productos",
|
209 |
+
"query": "SELECT \nCASE \n WHEN \"Descripcion\" LIKE '%QUESO%' OR \"Descripcion\" LIKE '%QUESO%' THEN 'Quesos'\n WHEN \"Descripcion\" LIKE '%LECHE%' OR \"Descripcion\" LIKE '%LACTEO%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%PAN%' OR \"Descripcion\" LIKE '%BOLLERIA%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%BEBIDA%' OR \"Descripcion\" LIKE '%REFRESCO%' THEN 'Bebidas'\n WHEN \"Descripcion\" LIKE '%EMBUTIDO%' OR \"Descripcion\" LIKE '%SALCHICHA%' THEN 'Embutidos'\n ELSE 'Otros'\nEND AS categoria,\nCOUNT(DISTINCT \"Descripcion\") AS variedad_productos\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY categoria\nORDER BY variedad_productos DESC\nLIMIT 3;"
|
210 |
+
}
|
211 |
+
],
|
212 |
+
"CLIENTES_CERO_TIEMPO": [
|
213 |
+
{
|
214 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Marzo 2025?",
|
215 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-03-31')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-03-31','-27 day') AND DATE('2025-03-31')\n);"
|
216 |
+
},
|
217 |
+
{
|
218 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en el primer trimestre de 2025?",
|
219 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-03-31')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-03-31','-27 day') AND DATE('2025-03-31')\n);"
|
220 |
+
},
|
221 |
+
{
|
222 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en el último semestre?",
|
223 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('now','-6 months')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('now','-6 months','-27 day') AND DATE('now','-6 months')\n);"
|
224 |
+
}
|
225 |
+
],
|
226 |
+
"CIUDAD_CERO_TIEMPO": [
|
227 |
+
{
|
228 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Guadalajara en Marzo 2025?",
|
229 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-03-31')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-03-31','-27 day') AND DATE('2025-03-31')\n);"
|
230 |
+
},
|
231 |
+
{
|
232 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Monterrey en Abril 2025?",
|
233 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-04-30')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-04-30','-27 day') AND DATE('2025-04-30')\n);"
|
234 |
+
},
|
235 |
+
{
|
236 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Zapopan en el último mes?",
|
237 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Zapopan%'\nAND \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('now','-1 month')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('now','-1 month','-27 day') AND DATE('now','-1 month')\n);"
|
238 |
+
}
|
239 |
+
],
|
240 |
+
"PRODUCTO_CIUDAD_TIEMPO": [
|
241 |
+
{
|
242 |
+
"pregunta": "¿Qué productos se vendieron más en Guadalajara en Marzo?",
|
243 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Descripcion\" IS NOT NULL\nAND strftime('%Y-%m', \"Fecha\") = '2025-03'\nGROUP BY \"Descripcion\"\nORDER BY total_vendido DESC\nLIMIT 5;"
|
244 |
+
},
|
245 |
+
{
|
246 |
+
"pregunta": "¿Cuál fue el producto más vendido en Monterrey en Abril?",
|
247 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Descripcion\" IS NOT NULL\nAND strftime('%Y-%m', \"Fecha\") = '2025-04'\nGROUP BY \"Descripcion\"\nORDER BY total_vendido DESC\nLIMIT 1;"
|
248 |
+
},
|
249 |
+
{
|
250 |
+
"pregunta": "Productos con mayor crecimiento en Zapopan el último trimestre",
|
251 |
+
"query": "SELECT \"Descripcion\", \n(SUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = strftime('%Y-%m','now') THEN \"Cantidad\" ELSE 0 END) - \nSUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = strftime('%Y-%m','now','-3 months') THEN \"Cantidad\" ELSE 0 END)) AS crecimiento\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Zapopan%'\nAND \"Descripcion\" IS NOT NULL\nGROUP BY \"Descripcion\"\nORDER BY crecimiento DESC\nLIMIT 5;"
|
252 |
+
}
|
253 |
+
],
|
254 |
+
"CLIENTE_CIUDAD_TIEMPO": [
|
255 |
+
{
|
256 |
+
"pregunta": "Clientes más valiosos de Guadalajara en el último trimestre",
|
257 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS valor_total\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Cliente\" IS NOT NULL\nAND \"Fecha\" BETWEEN DATE('now','-3 months') AND DATE('now')\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY valor_total DESC\nLIMIT 10;"
|
258 |
+
},
|
259 |
+
{
|
260 |
+
"pregunta": "Clientes nuevos en Monterrey durante Marzo",
|
261 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", MIN(DATE(\"Fecha\")) AS primera_compra\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Cliente\" IS NOT NULL\nAND strftime('%Y-%m', \"Fecha\") = '2025-03'\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING MIN(DATE(\"Fecha\")) >= DATE('2025-03-01')\nORDER BY primera_compra;"
|
262 |
+
},
|
263 |
+
{
|
264 |
+
"pregunta": "Clientes inactivos en Zapopan que compraron hace 6 meses",
|
265 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", MAX(DATE(\"Fecha\")) AS ultima_compra\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Zapopan%'\nAND \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now','-28 day')\n)\nAND DATE(\"Fecha\") BETWEEN DATE('now','-6 months') AND DATE('now','-5 months')\nGROUP BY \"Cliente\", \"Razon Social\";"
|
266 |
+
}
|
267 |
+
],
|
268 |
+
"TIEMPO_DINERO_CIUDAD": [
|
269 |
+
{
|
270 |
+
"pregunta": "Ventas totales por mes en Guadalajara",
|
271 |
+
"query": "SELECT strftime('%Y-%m', \"Fecha\") AS mes, SUM(\"Neto\") AS ventas_mensuales\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Fecha\" IS NOT NULL\nGROUP BY mes\nORDER BY mes DESC;"
|
272 |
+
},
|
273 |
+
{
|
274 |
+
"pregunta": "Crecimiento de ventas en Monterrey por trimestre",
|
275 |
+
"query": "SELECT \nCASE \n WHEN strftime('%m', \"Fecha\") BETWEEN '01' AND '03' THEN 'Q1'\n WHEN strftime('%m', \"Fecha\") BETWEEN '04' AND '06' THEN 'Q2'\n WHEN strftime('%m', \"Fecha\") BETWEEN '07' AND '09' THEN 'Q3'\n WHEN strftime('%m', \"Fecha\") BETWEEN '10' AND '12' THEN 'Q4'\nEND AS trimestre,\nSUM(\"Neto\") AS ventas_trimestrales\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Fecha\" IS NOT NULL\nGROUP BY trimestre\nORDER BY trimestre;"
|
276 |
+
},
|
277 |
+
{
|
278 |
+
"pregunta": "Comparativo de ventas: Guadalajara vs Monterrey por mes",
|
279 |
+
"query": "SELECT strftime('%Y-%m', \"Fecha\") AS mes,\nSUM(CASE WHEN \"Ciudad\" LIKE '%Guadalajara%' THEN \"Neto\" ELSE 0 END) AS ventas_guadalajara,\nSUM(CASE WHEN \"Ciudad\" LIKE '%Monterrey%' THEN \"Neto\" ELSE 0 END) AS ventas_monterrey\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL\nGROUP BY mes\nORDER BY mes DESC;"
|
280 |
+
}
|
281 |
+
]
|
282 |
+
}
|
intelligent_question_router.py
ADDED
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from supervised_classifier import QuestionClassifier
|
2 |
+
import torch
|
3 |
+
import json
|
4 |
+
|
5 |
+
class IntelligentQuestionRouter:
|
6 |
+
def __init__(self):
|
7 |
+
self.classifier = QuestionClassifier()
|
8 |
+
self._initialize_json()
|
9 |
+
|
10 |
+
def _initialize_json(self):
|
11 |
+
with open("examples.json", "r", encoding="utf-8") as f:
|
12 |
+
self.examples = json.load(f)
|
13 |
+
|
14 |
+
def _get_examples(self, category):
|
15 |
+
return self.examples.get(category, self.examples["CLIENTES_CERO"])
|
16 |
+
|
17 |
+
def route_question(self, question: str):
|
18 |
+
try:
|
19 |
+
ml_category_id = self.classifier.predict(question)
|
20 |
+
return self._get_examples(ml_category_id)
|
21 |
+
except Exception as e:
|
22 |
+
print("Error in routing question:", e)
|
23 |
+
return self._get_examples("CLIENTES_CERO")
|
24 |
+
|
25 |
+
|
load_json.py
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import json
|
2 |
+
def load_examples(path: str = 'examples.json'):
|
3 |
+
with open(path, "r", encoding="utf-8") as f:
|
4 |
+
return json.load(f)
|
5 |
+
|
6 |
+
examples = []
|
7 |
+
for category, items in data.items():
|
8 |
+
for item in items:
|
9 |
+
examples.append({
|
10 |
+
"category": category,
|
11 |
+
"question": item["pregunta"],
|
12 |
+
"sql": item["sql"]
|
13 |
+
})
|
14 |
+
return examples
|
memory.py
CHANGED
@@ -1,47 +1,26 @@
|
|
1 |
-
from
|
2 |
-
|
3 |
-
from
|
4 |
|
5 |
-
class ConversationMemory:
|
6 |
-
def __init__(self, max_history: int = Config.MAX_HISTORY):
|
7 |
-
self.history = deque(maxlen=max_history)
|
8 |
-
self.schema_cache = None
|
9 |
-
|
10 |
-
def add_interaction(self, question: str, sql: str, result: str):
|
11 |
-
self.history.append({
|
12 |
-
"question": question,
|
13 |
-
"sql": sql,
|
14 |
-
"result_summary": self._summarize_result(result)
|
15 |
-
})
|
16 |
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
else:
|
27 |
-
return f"Filas: {len(result)}, Columnas: {list(result.columns)}"
|
28 |
-
return str(result)
|
29 |
-
|
30 |
-
def get_context(self, current_question: str) -> str:
|
31 |
-
if not self.history:
|
32 |
-
return ""
|
33 |
-
last_relevant = []
|
34 |
-
for interaction in self.history:
|
35 |
-
if "producto" in interaction['question'].lower() and "producto" in current_question.lower():
|
36 |
-
last_relevant.append(interaction)
|
37 |
-
elif "cliente" in interaction['question'].lower() and "cliente" in current_question.lower():
|
38 |
-
last_relevant.append(interaction)
|
39 |
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
|
|
|
|
|
|
|
1 |
+
from langchain.vectorstores import Chroma
|
2 |
+
from langchain.embeddings import HuggingFaceEmbeddings
|
3 |
+
from langchain_core.documents import Document
|
4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
|
6 |
+
class Memory:
|
7 |
+
def __init__(self):
|
8 |
+
self.embedding_model = HuggingFaceEmbeddings(
|
9 |
+
model_name="sentence-transformers/all-MiniLM-L6-v2"
|
10 |
+
)
|
11 |
+
self.vector_store = Chroma(
|
12 |
+
persist_directory="./chroma_db", embedding_function=self.embedding_model
|
13 |
+
)
|
14 |
+
self.schema_cache = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
15 |
|
16 |
+
def add_interaction(self, question: str, answer: str, sql: str):
|
17 |
+
document = Document(
|
18 |
+
page_content=f"Pregunta: {question}\nRespuesta: {answer}\nSQL: {sql}",
|
19 |
+
metadata={"source": "interaction"},
|
20 |
+
)
|
21 |
+
self.vector_store.add_documents([document])
|
22 |
+
|
23 |
+
def get_relevant_interactions(self, question: str, top_k=3):
|
24 |
+
results = self.vector_store.similarity_search(question, k=top_k)
|
25 |
+
return "\n".join([d.page_content for d in results])
|
26 |
+
|
mojica_agent.py
CHANGED
@@ -1,18 +1,21 @@
|
|
1 |
-
from memory import
|
2 |
from config import Config
|
3 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
4 |
import torch, gc
|
5 |
import unicodedata
|
6 |
-
from typing import Dict, Tuple, Optional
|
7 |
import re
|
8 |
import pandas as pd
|
9 |
import sqlite3
|
|
|
10 |
|
11 |
|
12 |
class MojicaAgent:
|
13 |
def __init__(self, config: Config):
|
14 |
self.config = config
|
15 |
-
self.memory =
|
|
|
|
|
16 |
self.essential_columns = [
|
17 |
{
|
18 |
"name": "Descripcion",
|
@@ -34,172 +37,111 @@ class MojicaAgent:
|
|
34 |
},
|
35 |
{"name": "Neto", "type": "REAL", "description": "Valor neto de la venta"},
|
36 |
]
|
37 |
-
self.
|
38 |
-
self.
|
39 |
|
40 |
-
def
|
41 |
def try_load_model():
|
42 |
-
tokenizer = AutoTokenizer.from_pretrained(self.config.MODEL_NAME)
|
43 |
-
model = AutoModelForCausalLM.from_pretrained(
|
44 |
self.config.MODEL_NAME,
|
45 |
-
|
46 |
-
device_map="auto", # * <- Distribuye mejor entre GPU
|
47 |
torch_dtype="auto",
|
|
|
48 |
).eval()
|
49 |
-
return tokenizer, model
|
50 |
|
51 |
try:
|
52 |
-
|
53 |
except torch.cuda.OutOfMemoryError:
|
54 |
-
# * Libera memoria e intenta de nuevo
|
55 |
gc.collect()
|
56 |
torch.cuda.empty_cache()
|
57 |
torch.cuda.ipc_collect()
|
58 |
-
|
59 |
|
60 |
-
def
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
{"
|
66 |
]
|
67 |
-
|
68 |
-
|
69 |
-
|
|
|
70 |
|
71 |
-
def
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
return f"""
|
76 |
-
SELECT DISTINCT "Cliente", "Razon Social"
|
77 |
-
FROM "{table}"
|
78 |
-
WHERE "Cliente" IS NOT NULL
|
79 |
-
AND "Cliente" NOT IN (
|
80 |
-
SELECT DISTINCT "Cliente"
|
81 |
-
FROM "{table}"
|
82 |
-
WHERE DATE("Fecha") >= DATE('now', '-28 day')
|
83 |
-
);
|
84 |
-
"""
|
85 |
-
return None
|
86 |
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
key = candidate.lower()
|
91 |
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
if corrected:
|
96 |
-
return f'"{corrected}"'
|
97 |
|
98 |
-
|
99 |
-
|
100 |
-
if corrected:
|
101 |
-
return f'"{corrected}"'
|
102 |
|
103 |
-
|
|
|
104 |
|
105 |
-
|
106 |
-
cursor
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
"Sales": "sells",
|
117 |
-
}
|
118 |
-
# * Opcional (convierte la key en minuscula solo si el valor esta en real_columns)
|
119 |
-
alias_map = {
|
120 |
-
key.lower(): value
|
121 |
-
for key, value in aliases.items()
|
122 |
-
if value in real_columns
|
123 |
-
}
|
124 |
-
pattern = r'"(\w+)"'
|
125 |
|
126 |
-
|
127 |
-
|
|
|
|
|
128 |
|
129 |
-
def
|
130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
131 |
table_name = self.schema["table_name"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
132 |
|
133 |
-
# * Detectamos el tipo de pregunta para asignar los ejemplos
|
134 |
-
question_type = (
|
135 |
-
"PRODUCTOS"
|
136 |
-
if "producto" in question.lower()
|
137 |
-
else "CLIENTES" if "cliente" in question.lower() else "GENERAL"
|
138 |
-
)
|
139 |
-
examples = {
|
140 |
-
"PRODUCTOS": (
|
141 |
-
"-- P: 'Top 10 productos más vendidos'\n"
|
142 |
-
'SELECT "Descripcion", SUM("Cantidad") AS total_vendido\n'
|
143 |
-
f'FROM "{table_name}"\n'
|
144 |
-
'WHERE "Descripcion" IS NOT NULL\n'
|
145 |
-
'GROUP BY "Descripcion"\n'
|
146 |
-
"ORDER BY total_vendido DESC\n"
|
147 |
-
"LIMIT 10;\n\n"
|
148 |
-
"-- P: 'Productos con mayor valor neto'\n"
|
149 |
-
'SELECT "Descripcion", SUM("Neto") AS valor_total\n'
|
150 |
-
f'FROM "{table_name}"\n'
|
151 |
-
'WHERE "Descripcion" IS NOT NULL\n'
|
152 |
-
'GROUP BY "Descripcion"\n'
|
153 |
-
"ORDER BY valor_total DESC\n"
|
154 |
-
"LIMIT 5;"
|
155 |
-
),
|
156 |
-
"CLIENTES": (
|
157 |
-
"-- P: 'Top 5 clientes con mayor valor neto'\n"
|
158 |
-
'SELECT "Cliente", SUM("Neto") AS valor_total\n'
|
159 |
-
f'FROM "{table_name}"\n'
|
160 |
-
"WHERE \"Cliente\" IS NOT NULL AND \"Fecha\" BETWEEN '2025-01-01' AND '2025-12-31'\n"
|
161 |
-
'GROUP BY "Cliente"\n'
|
162 |
-
"ORDER BY valor_total DESC\n"
|
163 |
-
"LIMIT 5;\n\n"
|
164 |
-
"-- P: 'Clientes con más compras en marzo'\n"
|
165 |
-
'SELECT "Cliente", COUNT(*) AS total_compras\n'
|
166 |
-
f'FROM "{table_name}"\n'
|
167 |
-
"WHERE \"Cliente\" IS NOT NULL AND strftime('%m', \"Fecha\") = '03'\n"
|
168 |
-
'GROUP BY "Cliente"\n'
|
169 |
-
"ORDER BY total_compras DESC\n"
|
170 |
-
"LIMIT 10;\n\n"
|
171 |
-
"-- P: 'Clientes de Guadalajara con más compras'\n"
|
172 |
-
'SELECT "Cliente", "Razon Social", COUNT(*) AS total_compras\n'
|
173 |
-
f'FROM "{table_name}"\n'
|
174 |
-
'WHERE "Cliente" IS NOT NULL AND "Ciudad" = \'Guadalajara\'\n'
|
175 |
-
'GROUP BY "Cliente", "Razon Social"\n'
|
176 |
-
"ORDER BY total_compras DESC\n"
|
177 |
-
"LIMIT 10;"
|
178 |
-
),
|
179 |
-
"GENERAL": (
|
180 |
-
"-- P: 'Ventas totales por mes'\n"
|
181 |
-
'SELECT strftime(\'%m\', "Fecha") AS mes, SUM("Neto") AS ventas\n'
|
182 |
-
f'FROM "{table_name}"\n'
|
183 |
-
"WHERE mes IS NOT NULL\n"
|
184 |
-
"GROUP BY mes\n"
|
185 |
-
"ORDER BY mes;\n\n"
|
186 |
-
"-- P: 'Producto menos vendido en 2025'\n"
|
187 |
-
'SELECT "Descripcion", SUM("Cantidad") AS total_vendido\n'
|
188 |
-
f'FROM "{table_name}"\n'
|
189 |
-
"WHERE \"Descripcion\" IS NOT NULL AND \"Fecha\" BETWEEN '2025-01-01' AND '2025-12-31'\n"
|
190 |
-
'GROUP BY "Descripcion"\n'
|
191 |
-
"ORDER BY total_vendido ASC\n"
|
192 |
-
"LIMIT 1;"
|
193 |
-
),
|
194 |
-
}
|
195 |
-
# * Retornamos el prompt
|
196 |
return (
|
197 |
f"""
|
198 |
-
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
+ "\n".join(
|
204 |
[
|
205 |
f"- {col['name']} ({col['type']}): {col['description']}"
|
@@ -207,114 +149,161 @@ class MojicaAgent:
|
|
207 |
]
|
208 |
)
|
209 |
+ f"""
|
210 |
-
|
211 |
-
### CONTEXTO (Últimas interacciones) ###
|
212 |
-
{memory_context if memory_context else "Sin historial relevante"}
|
213 |
-
|
214 |
-
|
215 |
-
### EJEMPLOS ({question_type}) ###
|
216 |
-
{examples[question_type]}
|
217 |
-
|
218 |
-
### REGLAS CRÍTICAS ###
|
219 |
-
- Usar siempre nombres exactos de columnas
|
220 |
-
- Usar solo las columnas listadas
|
221 |
-
- Prohibido inventar columnas
|
222 |
-
- Para el nombre del cliente, usar SIEMPRE "Razon Social".
|
223 |
-
- Para un mes específico usar: strftime('%m', "Fecha") = 'MM'
|
224 |
-
- Para cantidades usar SUM("Cantidad"), para dinero usar SUM("Neto")
|
225 |
-
- Para ciudad usar SIEMPRE "Ciudad"
|
226 |
-
- Agrupar por la dimensión principal (producto/cliente)
|
227 |
-
- Ordenar DESC para 'más/mayor', ASC para 'menos/menor'
|
228 |
-
- Usar LIMIT para top N
|
229 |
-
- Contesta siempre en el idioma en el que se te pregunta no traduzcas.
|
230 |
-
- Año actual: 2025
|
231 |
-
- Siempre terminar con un LIMIT = 1 en caso que se indique lo contrario
|
232 |
-
- Para 'más vendido' usar SUM("Cantidad"), para 'mayor valor' usar SUM("Neto")
|
233 |
-
- Usar "Razon Social" cuando pregunten por el nombre del cliente
|
234 |
-
- Usar "Ciudad" para filtrar o agrupar por ubicación
|
235 |
-
- Queda estrictamente prohibido usar acentos
|
236 |
-
- **Siempre excluir valores nulos con 'IS NOT NULL' en las columnas usadas en WHERE, GROUP BY u ORDER BY**
|
237 |
-
|
238 |
-
### PREGUNTA ACTUAL ###
|
239 |
-
\"\"\"{question}\"\"\"
|
240 |
|
241 |
-
|
242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
243 |
)
|
244 |
|
245 |
-
def
|
246 |
-
|
247 |
-
|
248 |
-
|
249 |
-
|
250 |
-
|
251 |
-
|
252 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
253 |
|
254 |
-
|
|
|
255 |
sql_matches = list(
|
256 |
re.finditer(
|
257 |
r"(SELECT|WITH|INSERT|UPDATE|DELETE)[\s\S]+?;", output, re.IGNORECASE
|
258 |
)
|
259 |
)
|
260 |
|
261 |
-
# * si no existe nada entonces retornamos None
|
262 |
if not sql_matches:
|
263 |
return None
|
264 |
|
265 |
-
#
|
266 |
-
sql = sql_matches[-1].group(0).strip()
|
267 |
|
268 |
-
#
|
269 |
if any(
|
270 |
-
|
271 |
-
for
|
272 |
):
|
273 |
return None
|
274 |
-
|
|
|
275 |
if not sql.endswith(";"):
|
276 |
sql += ";"
|
277 |
|
278 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
279 |
sql = remove_accents(sql)
|
280 |
|
281 |
-
#
|
282 |
-
|
283 |
-
|
|
|
|
|
|
|
|
|
284 |
|
285 |
-
# * SQL limpio mas no corregido de erores, corregumos los errores y despues enviamos
|
286 |
validate_sql = self._validate_and_correct_sql(sql)
|
287 |
return validate_sql
|
288 |
|
289 |
-
def
|
290 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
291 |
try:
|
292 |
-
|
293 |
-
connection.close()
|
294 |
-
return result
|
295 |
except Exception as e:
|
296 |
-
return f"Error
|
297 |
-
|
298 |
-
|
299 |
-
|
300 |
-
def consult(self, question: str) -> Tuple[str, any]:
|
301 |
-
if self._handle_speecial_cases(question=question) != None:
|
302 |
-
sql_query = self._handle_speecial_cases(question=question)
|
303 |
-
result = self._execute_sql(sql_query)
|
304 |
-
# self.memory.add_interaction(question, sql_query, result)
|
305 |
-
return sql_query, result
|
306 |
-
|
307 |
-
prompt = self._build_prompt(question)
|
308 |
tokenized_input = self.tokenizer(
|
309 |
-
|
310 |
return_tensors="pt",
|
311 |
truncation=True,
|
312 |
max_length=self.config.MAX_TOKENS,
|
313 |
).to(self.config.DEVICE)
|
314 |
-
|
315 |
-
# * Desactiva el cálculo de gradientes -> Siempre poner cuando se haga prediccion
|
316 |
-
# * - Reduce consumo de memoria
|
317 |
-
# * - Acelera inferencia
|
318 |
with torch.no_grad():
|
319 |
tokenized_output_model = self.model.generate(
|
320 |
**tokenized_input,
|
@@ -326,17 +315,33 @@ class MojicaAgent:
|
|
326 |
do_sample=True,
|
327 |
pad_token_id=self.tokenizer.eos_token_id,
|
328 |
)
|
329 |
-
|
330 |
output_model = self.tokenizer.decode(
|
331 |
tokenized_output_model[0], skip_special_tokens=True
|
332 |
)
|
|
|
333 |
|
334 |
-
|
335 |
-
|
336 |
-
|
337 |
-
|
338 |
-
|
339 |
-
|
340 |
-
|
341 |
-
|
342 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from memory import Memory as VectorMemory
|
2 |
from config import Config
|
3 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
4 |
import torch, gc
|
5 |
import unicodedata
|
6 |
+
from typing import Dict, Tuple, Optional, Any
|
7 |
import re
|
8 |
import pandas as pd
|
9 |
import sqlite3
|
10 |
+
from intelligent_question_router import IntelligentQuestionRouter
|
11 |
|
12 |
|
13 |
class MojicaAgent:
|
14 |
def __init__(self, config: Config):
|
15 |
self.config = config
|
16 |
+
self.memory = VectorMemory()
|
17 |
+
self.router = IntelligentQuestionRouter()
|
18 |
+
self._load_training_data()
|
19 |
self.essential_columns = [
|
20 |
{
|
21 |
"name": "Descripcion",
|
|
|
37 |
},
|
38 |
{"name": "Neto", "type": "REAL", "description": "Valor neto de la venta"},
|
39 |
]
|
40 |
+
self._initialize_database()
|
41 |
+
self._initialize_model()
|
42 |
|
43 |
+
def _initialize_model(self):
|
44 |
def try_load_model():
|
45 |
+
self.tokenizer = AutoTokenizer.from_pretrained(self.config.MODEL_NAME)
|
46 |
+
self.model = AutoModelForCausalLM.from_pretrained(
|
47 |
self.config.MODEL_NAME,
|
48 |
+
device_map="auto",
|
|
|
49 |
torch_dtype="auto",
|
50 |
+
trust_remote_code=True,
|
51 |
).eval()
|
|
|
52 |
|
53 |
try:
|
54 |
+
try_load_model()
|
55 |
except torch.cuda.OutOfMemoryError:
|
|
|
56 |
gc.collect()
|
57 |
torch.cuda.empty_cache()
|
58 |
torch.cuda.ipc_collect()
|
59 |
+
try_load_model()
|
60 |
|
61 |
+
def _load_training_data(self):
|
62 |
+
training_examples = [
|
63 |
+
{"question": "productos más vendidos", "category": "producto"},
|
64 |
+
{"question": "mejor producto", "category": "producto"},
|
65 |
+
{"question": "clientes que más compran", "category": "cliente"},
|
66 |
+
{"question": "clientes inactivos", "category": "cliente"},
|
67 |
]
|
68 |
+
try:
|
69 |
+
self.router.semantic_classifier.train(training_examples)
|
70 |
+
except Exception as e:
|
71 |
+
print(f"Error training semantic classifier: {e}")
|
72 |
|
73 |
+
def _validate_result_existing(self, result):
|
74 |
+
# Si es un string de error
|
75 |
+
if isinstance(result, str) and "Error" in result:
|
76 |
+
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
77 |
|
78 |
+
# Si es un DataFrame vacío
|
79 |
+
if hasattr(result, "empty") and result.empty:
|
80 |
+
return False
|
|
|
81 |
|
82 |
+
# Si es una lista vacía
|
83 |
+
if isinstance(result, list) and len(result) == 0:
|
84 |
+
return False
|
|
|
|
|
85 |
|
86 |
+
# En cualquier otro caso, asumimos éxito
|
87 |
+
return True
|
|
|
|
|
88 |
|
89 |
+
def _initialize_database(self):
|
90 |
+
self.conn = sqlite3.connect(self.config.DB_PATH)
|
91 |
|
92 |
+
cursor = self.conn.cursor()
|
93 |
+
cursor.execute(f"DROP TABLE IF EXISTS {self.config.TABLE_NAME}")
|
94 |
+
self.conn.commit()
|
95 |
+
df = pd.read_csv(self.config.CSV_PATH, low_memory=False)
|
96 |
+
|
97 |
+
real_cols = [
|
98 |
+
col["name"] for col in self.essential_columns if col["type"] == "REAL"
|
99 |
+
]
|
100 |
+
for col in real_cols:
|
101 |
+
if col in df.columns:
|
102 |
+
df[col] = pd.to_numeric(df[col], errors="coerce")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
|
104 |
+
df.to_sql(self.config.TABLE_NAME, self.conn, if_exists="replace", index=False)
|
105 |
+
self.schema = self._get_schema_structured()
|
106 |
+
# Configuracion de pandas:
|
107 |
+
pd.set_option("display.float_format", "{:,.2f}".format)
|
108 |
|
109 |
+
def _get_schema_structured(self) -> Dict:
|
110 |
+
if self.memory.schema_cache:
|
111 |
+
return self.memory.schema_cache
|
112 |
+
cursor = self.conn.cursor()
|
113 |
+
cursor.execute(f"PRAGMA table_info({self.config.TABLE_NAME})")
|
114 |
+
columns = [
|
115 |
+
{"name": column[1], "type": column[2]} for column in cursor.fetchall()
|
116 |
+
]
|
117 |
+
schema = {"table_name": self.config.TABLE_NAME, "columns": columns}
|
118 |
+
self.memory.schema_cache = schema
|
119 |
+
return schema
|
120 |
+
|
121 |
+
def _generate_sql_prompt(self, question: str) -> str:
|
122 |
+
memory_context = self.memory.get_relevant_memory(question)
|
123 |
table_name = self.schema["table_name"]
|
124 |
+
# Uso del router
|
125 |
+
try:
|
126 |
+
examples_list = self.router.route_question(question)
|
127 |
+
# Convertir ejemplos a texto para el prompt
|
128 |
+
examples_text = "\n".join(
|
129 |
+
[f"-- P: '{ex['pregunta']}'\n{ex['query']}\n" for ex in examples_list]
|
130 |
+
)
|
131 |
+
question_type = "ROUTED_EXAMPLES"
|
132 |
+
except Exception as e:
|
133 |
+
print(f"Router failed, using manual detection: {e}")
|
134 |
+
# Fallback a detección manual
|
135 |
+
# question_type = self._detect_question_type_manual(question)
|
136 |
+
# examples_text = self.examples.get(question_type, "")
|
137 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
138 |
return (
|
139 |
f"""
|
140 |
+
### TAREA ###
|
141 |
+
Generar SOLO código SQL para la pregunta, usando EXCLUSIVAMENTE la tabla: "{table_name}"
|
142 |
+
|
143 |
+
### COLUMNAS RELEVANTES ###
|
144 |
+
"""
|
145 |
+ "\n".join(
|
146 |
[
|
147 |
f"- {col['name']} ({col['type']}): {col['description']}"
|
|
|
149 |
]
|
150 |
)
|
151 |
+ f"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
|
153 |
+
### CONTEXTO (Últimas interacciones) ###
|
154 |
+
{memory_context if memory_context else "Sin historial relevante"}
|
155 |
+
|
156 |
+
### EJEMPLOS ###
|
157 |
+
{examples_text}
|
158 |
+
|
159 |
+
### REGLAS CRÍTICAS ###
|
160 |
+
- Usar siempre nombres exactos de columnas
|
161 |
+
- Usar solo las columnas listadas
|
162 |
+
- Prohibido inventar columnas
|
163 |
+
- Para el nombre del cliente, usar SIEMPRE "Razon Social".
|
164 |
+
- Para un mes específico usar: strftime('%m', "Fecha") = 'MM'
|
165 |
+
- Para cantidades usar SUM("Cantidad"), para dinero usar SUM("Neto")
|
166 |
+
- Agrupar por la dimensión principal (producto/cliente)
|
167 |
+
- Ordenar DESC para 'más/mayor', ASC para 'menos/menor'
|
168 |
+
- Contesta siempre en el idioma en el que se te pregunta no traduzcas.
|
169 |
+
- Año actual: 2025
|
170 |
+
- No inventes columnas o tablas que no existan
|
171 |
+
- Para preguntas sobre clientes cero, SIEMPRE usar la subconsulta NOT IN con las últimas 4 semanas.
|
172 |
+
- Si se menciona una ciudad, incluir el filtro AND "Ciudad" LIKE '%...%'
|
173 |
+
- Usa LIMIT cuando se te pida un numero finito de datos
|
174 |
+
- Para 'más vendido' usar SUM("Cantidad"), para 'mayor valor' usar SUM("Neto")
|
175 |
+
- Usar "Razon Social" cuando pregunten por el nombre del cliente
|
176 |
+
- Usar "Ciudad" para filtrar o agrupar por ubicación
|
177 |
+
- Queda estrictamente prohibido usar acentos
|
178 |
+
- **Siempre excluir valores nulos con 'IS NOT NULL' en las columnas usadas en WHERE, GROUP BY u ORDER BY**
|
179 |
+
- Para preguntas sobre ciudad SIEMPRE incluir "Ciudad" en la query
|
180 |
+
- Para busquedas por Descripcion siempre usar LIKE
|
181 |
+
- Mandar solo la cantidad de rows que el usuario pide.
|
182 |
+
### PREGUNTA ACTUAL ###
|
183 |
+
\"\"\"{question}\"\"\"
|
184 |
+
|
185 |
+
### SQL:
|
186 |
+
"""
|
187 |
)
|
188 |
|
189 |
+
def _generate_analysis_prompt(self, question: str, result: Any) -> str:
|
190 |
+
return f"""
|
191 |
+
Basado EXCLUSIVAMENTE en estos datos: {result}
|
192 |
+
|
193 |
+
Responde esta pregunta: {question}
|
194 |
+
|
195 |
+
Reglas estrictas:
|
196 |
+
- Nunca inventes numeros
|
197 |
+
- Usa solo datos proporcionados
|
198 |
+
- Maximo una oracion
|
199 |
+
"""
|
200 |
+
|
201 |
+
def _clean_analysis_output(self, ouput: str) -> Optional[str]:
|
202 |
+
pattern = r"Respuesta:([\s\S]+)"
|
203 |
+
match = re.search(pattern, ouput)
|
204 |
+
if match:
|
205 |
+
return match.group(1).strip()
|
206 |
+
else:
|
207 |
+
return "Sin análisis"
|
208 |
|
209 |
+
def _clean_sql_output(self, output: str) -> Optional[str]:
|
210 |
+
# Encuentra todas las posibles queries completas que terminen en ;
|
211 |
sql_matches = list(
|
212 |
re.finditer(
|
213 |
r"(SELECT|WITH|INSERT|UPDATE|DELETE)[\s\S]+?;", output, re.IGNORECASE
|
214 |
)
|
215 |
)
|
216 |
|
|
|
217 |
if not sql_matches:
|
218 |
return None
|
219 |
|
220 |
+
# Tomar la última query encontrada
|
221 |
+
sql = sql_matches[-1].group(0).strip()
|
222 |
|
223 |
+
# Seguridad: bloquear queries peligrosas
|
224 |
if any(
|
225 |
+
cmd in sql.upper()
|
226 |
+
for cmd in ["DROP", "DELETE", "UPDATE", "INSERT", "ALTER"]
|
227 |
):
|
228 |
return None
|
229 |
+
|
230 |
+
# Asegurar que termine en ;
|
231 |
if not sql.endswith(";"):
|
232 |
sql += ";"
|
233 |
|
234 |
+
# ────────────────────────────────
|
235 |
+
# 1. Quitar acentos de toda la query
|
236 |
+
# ────────────────────────────────
|
237 |
+
def remove_accents(text: str) -> str:
|
238 |
+
return "".join(
|
239 |
+
c
|
240 |
+
for c in unicodedata.normalize("NFKD", text)
|
241 |
+
if not unicodedata.combining(c)
|
242 |
+
)
|
243 |
+
|
244 |
sql = remove_accents(sql)
|
245 |
|
246 |
+
# ────────────────────────────────
|
247 |
+
# 2. Agregar LIMIT si no existe
|
248 |
+
# ────────────────────────────────
|
249 |
+
# Buscar si ya hay un LIMIT en la query
|
250 |
+
# if not re.search(r"\bLIMIT\s+\d+", sql, re.IGNORECASE):
|
251 |
+
# # Insertar antes del último punto y coma
|
252 |
+
# sql = sql[:-1] + " LIMIT 1;" # puedes cambiar 100 por el valor default que quieras
|
253 |
|
|
|
254 |
validate_sql = self._validate_and_correct_sql(sql)
|
255 |
return validate_sql
|
256 |
|
257 |
+
def _validate_and_correct_sql(self, sql: str) -> str:
|
258 |
+
cur = self.conn.cursor()
|
259 |
+
cur.execute(f'PRAGMA table_info("{self.config.TABLE_NAME}")')
|
260 |
+
real_columns = [row[1] for row in cur.fetchall()]
|
261 |
+
column_lower_map = {col.lower(): col for col in real_columns}
|
262 |
+
aliases = {
|
263 |
+
"city": "Ciudad",
|
264 |
+
"client": "Cliente",
|
265 |
+
"razon_social": "Razon Social",
|
266 |
+
"razón social": "Razon Social",
|
267 |
+
"Sales": "sells",
|
268 |
+
'"Date"': "Fecha",
|
269 |
+
"mojica_Clientes": "sells",
|
270 |
+
"value_total": "valor_total",
|
271 |
+
"strstrftime": "strftime",
|
272 |
+
}
|
273 |
+
alias_map = {k.lower(): v for k, v in aliases.items()}
|
274 |
+
|
275 |
+
pattern = r"\b\w+\b"
|
276 |
+
|
277 |
+
def replace_column(m):
|
278 |
+
candidate = m.group(0) # Palabra encontrada
|
279 |
+
key = candidate.lower()
|
280 |
+
# ¿Es una columna?
|
281 |
+
corrected = column_lower_map.get(key)
|
282 |
+
if corrected:
|
283 |
+
return corrected
|
284 |
+
|
285 |
+
# ¿Es una alias?
|
286 |
+
corrected = alias_map.get(key)
|
287 |
+
if corrected is not None:
|
288 |
+
return corrected
|
289 |
+
return candidate # si no encuentra nada, lo deja igual
|
290 |
+
|
291 |
+
return re.sub(pattern, replace_column, sql).replace("\\", "")
|
292 |
+
|
293 |
+
def _execute_sql(self, sql: str) -> Any:
|
294 |
try:
|
295 |
+
return pd.read_sql_query(sql, self.conn)
|
|
|
|
|
296 |
except Exception as e:
|
297 |
+
return f"Error: {str(e)}"
|
298 |
+
|
299 |
+
def consult(self, question: str) -> Tuple[str, Any, str]:
|
300 |
+
sql_prompt = self._generate_sql_prompt(question)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
301 |
tokenized_input = self.tokenizer(
|
302 |
+
sql_prompt,
|
303 |
return_tensors="pt",
|
304 |
truncation=True,
|
305 |
max_length=self.config.MAX_TOKENS,
|
306 |
).to(self.config.DEVICE)
|
|
|
|
|
|
|
|
|
307 |
with torch.no_grad():
|
308 |
tokenized_output_model = self.model.generate(
|
309 |
**tokenized_input,
|
|
|
315 |
do_sample=True,
|
316 |
pad_token_id=self.tokenizer.eos_token_id,
|
317 |
)
|
|
|
318 |
output_model = self.tokenizer.decode(
|
319 |
tokenized_output_model[0], skip_special_tokens=True
|
320 |
)
|
321 |
+
sql = self._clean_sql_output(output_model)
|
322 |
|
323 |
+
# * Ejecución de SQL y generación de analisis
|
324 |
+
result = self._execute_sql(sql)
|
325 |
+
# * INICIO DE ANALISIS (COMENTADO)
|
326 |
+
# Analisis
|
327 |
+
# analysis_prompt = self._generate_analysis_prompt(question, result)
|
328 |
+
# analyzed_token_input = self.tokenizer(
|
329 |
+
# analysis_prompt,
|
330 |
+
# return_tensors="pt",
|
331 |
+
# truncation=True,
|
332 |
+
# max_length=self.config.MAX_TOKENS,
|
333 |
+
# ).to(self.config.DEVICE)
|
334 |
+
# with torch.no_grad():
|
335 |
+
# tokenized_analysis_output_model = self.model.generate(
|
336 |
+
# **analyzed_token_input,
|
337 |
+
# max_new_tokens=self.config.MAX_NEW_TOKENS,
|
338 |
+
# temperature=0.65,
|
339 |
+
# )
|
340 |
+
# analysis = self.tokenizer.decode(
|
341 |
+
# tokenized_analysis_output_model[0], skip_special_tokens=True
|
342 |
+
# )
|
343 |
+
# analysis = self._clean_analysis_output(analysis)
|
344 |
+
# analysis <- LE quite ese parametro
|
345 |
+
# * FIN DE ANALISIS (COMENTADO)
|
346 |
+
self.memory.add_interaction(question=question, answer=result, sql=sql)
|
347 |
+
return sql, result
|
requirements.txt
CHANGED
Binary files a/requirements.txt and b/requirements.txt differ
|
|
semantic_classifier.py
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from sentence_transformers import SentenceTransformer
|
2 |
+
from sklearn.cluster import KMeans
|
3 |
+
from config import Config
|
4 |
+
from load_json import load_examples
|
5 |
+
|
6 |
+
from sentence_transformers import SentenceTransformer
|
7 |
+
from sklearn.cluster import KMeans
|
8 |
+
|
9 |
+
class SemanticClassifier:
|
10 |
+
def __init__(self, model_name="paraphrase-multilingual-MiniLM-L12-v2", initialized_train=True):
|
11 |
+
self.model = SentenceTransformer(model_name)
|
12 |
+
self.clusters = {}
|
13 |
+
self.examples_embeddings = None
|
14 |
+
self.kmeans = None
|
15 |
+
if initialized_train:
|
16 |
+
self.train()
|
17 |
+
|
18 |
+
def train(self, train_data=Config.EXMAPLES_JSON, n_clusters=15):
|
19 |
+
examples = load_examples(train_data)
|
20 |
+
|
21 |
+
# * Aplanar ejemplos
|
22 |
+
flat_examples = []
|
23 |
+
for category, items in examples.items():
|
24 |
+
for item in items:
|
25 |
+
flat_examples.append({
|
26 |
+
"category": category,
|
27 |
+
"pregunta": item["pregunta"],
|
28 |
+
"query": item["query"]
|
29 |
+
})
|
30 |
+
|
31 |
+
questions = [ex["pregunta"] for ex in flat_examples]
|
32 |
+
|
33 |
+
# * Obtener embeddings
|
34 |
+
embeddings = self.model.encode(questions)
|
35 |
+
|
36 |
+
# * Clustering
|
37 |
+
self.kmeans = KMeans(n_clusters=n_clusters, random_state=12)
|
38 |
+
cluster_ids = self.kmeans.fit_predict(embeddings)
|
39 |
+
|
40 |
+
# * Guardar ejemplos por cluster
|
41 |
+
for i, cluster_id in enumerate(cluster_ids):
|
42 |
+
# * Crear lista si no existe
|
43 |
+
if cluster_id not in self.clusters:
|
44 |
+
self.clusters[cluster_id] = []
|
45 |
+
# * Agregamos el ejemplo
|
46 |
+
self.clusters[cluster_id].append(flat_examples[i])
|
47 |
+
self.examples_embeddings = embeddings
|
48 |
+
|
49 |
+
def classify(self, question: str):
|
50 |
+
# * En formato de embedding
|
51 |
+
question_embedding = self.model.encode([question])
|
52 |
+
# * Encontrar el cluster más cercano
|
53 |
+
cluster_id = self.kmeans.predict(question_embedding)[0]
|
54 |
+
# * Retornamos los ejemplos de ese cluster
|
55 |
+
return self.clusters.get(cluster_id, [])
|
56 |
+
|
57 |
+
|
58 |
+
# * FORMA DE USARSE
|
59 |
+
# classifier = SemanticClassifier()
|
60 |
+
# classifier.train(Config.EXMAPLES_JSON, n_clusters=5)
|
61 |
+
|
62 |
+
# resultado = classifier.classify("¿Cuantas ciudades tenemos registradas?")
|
63 |
+
# print(resultado) # te devuelve ejemplos de ese cluster
|
supervised_classifier.py
ADDED
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from transformers import (
|
2 |
+
AutoTokenizer,
|
3 |
+
AutoModelForSequenceClassification,
|
4 |
+
Trainer,
|
5 |
+
TrainingArguments,
|
6 |
+
)
|
7 |
+
from config import Config
|
8 |
+
import json
|
9 |
+
from datasets import Dataset
|
10 |
+
import torch
|
11 |
+
|
12 |
+
|
13 |
+
class QuestionClassifier:
|
14 |
+
def __init__(
|
15 |
+
self, model_name="distilbert-base-multilingual-cased", initialized_train=True
|
16 |
+
):
|
17 |
+
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
18 |
+
self.model_name = model_name
|
19 |
+
self.category2id = None
|
20 |
+
self.category2id = None
|
21 |
+
if initialized_train:
|
22 |
+
self.train()
|
23 |
+
|
24 |
+
def train(self, json_path=Config.EXMAPLES_JSON, num_epochs=3):
|
25 |
+
# * Cargar ejemplos
|
26 |
+
with open(json_path, "r", encoding="utf-8") as f:
|
27 |
+
examples = json.load(f)
|
28 |
+
texts, labels, category2id = self._prepare_supervised_data(examples)
|
29 |
+
self.category2id = category2id
|
30 |
+
self.id2category = {value: key for key, value in category2id.items()}
|
31 |
+
self.model = AutoModelForSequenceClassification.from_pretrained(
|
32 |
+
self.model_name, num_labels=len(category2id)
|
33 |
+
)
|
34 |
+
|
35 |
+
encodings = self.tokenizer(texts, truncation=True, padding=True)
|
36 |
+
dataset = Dataset.from_dict(
|
37 |
+
{
|
38 |
+
"input_ids": encodings["input_ids"],
|
39 |
+
"attention_mask": encodings["attention_mask"],
|
40 |
+
"labels": labels,
|
41 |
+
}
|
42 |
+
)
|
43 |
+
|
44 |
+
training_args = TrainingArguments(
|
45 |
+
output_dir="./results",
|
46 |
+
per_device_train_batch_size=8,
|
47 |
+
num_train_epochs=num_epochs,
|
48 |
+
logging_steps=1,
|
49 |
+
# logging_strategy="steps",
|
50 |
+
report_to="none",
|
51 |
+
save_strategy="no",
|
52 |
+
remove_unused_columns=False,
|
53 |
+
eval_strategy="no",
|
54 |
+
)
|
55 |
+
|
56 |
+
# 4. Trainer
|
57 |
+
trainer = Trainer(model=self.model, args=training_args, train_dataset=dataset)
|
58 |
+
|
59 |
+
trainer.train()
|
60 |
+
|
61 |
+
def _prepare_supervised_data(self, examples):
|
62 |
+
category2id = {cat: i for i, cat in enumerate(examples.keys())}
|
63 |
+
texts = []
|
64 |
+
labels = []
|
65 |
+
for category, items in examples.items():
|
66 |
+
for item in items:
|
67 |
+
texts.append(item["pregunta"])
|
68 |
+
labels.append(category2id[category])
|
69 |
+
return texts, labels, category2id
|
70 |
+
|
71 |
+
def predict(self, question: str):
|
72 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
73 |
+
self.model.to(device)
|
74 |
+
|
75 |
+
inputs = self.tokenizer(
|
76 |
+
question, return_tensors="pt", truncation=True, padding=True
|
77 |
+
)
|
78 |
+
|
79 |
+
inputs = {k: v.to(device) for k, v in inputs.items()}
|
80 |
+
|
81 |
+
with torch.no_grad():
|
82 |
+
outputs = self.model(**inputs)
|
83 |
+
predicted_class_id = outputs.logits.argmax().item()
|
84 |
+
|
85 |
+
return self.id2category[predicted_class_id]
|
86 |
+
|
87 |
+
|
88 |
+
# * FORMA DE USARSE
|
89 |
+
# qc = QuestionClassifier()
|
90 |
+
# qc.train()
|
91 |
+
|
92 |
+
# categoria = qc.predict("Dame los productos más vendidos")
|
93 |
+
# print(categoria) # → 'PRODUCTOS'
|