Spaces:
Sleeping
Sleeping
Carlos Isael Ramírez González
commited on
Commit
·
56ff037
1
Parent(s):
a3373fd
Modelo nuevo completado
Browse files- app.py +15 -9
- config.py +3 -2
- examples.json +282 -0
- intelligent_question_router.py +25 -0
- load_json.py +14 -0
- memory.py +23 -44
- mojica_agent.py +234 -229
- requirements.txt +0 -0
- semantic_classifier.py +63 -0
- supervised_classifier.py +93 -0
app.py
CHANGED
|
@@ -9,33 +9,39 @@ app = FastAPI()
|
|
| 9 |
|
| 10 |
mojica_bot = MojicaAgent(Config)
|
| 11 |
|
|
|
|
| 12 |
# * Esquema de entrada como marshmellow
|
| 13 |
-
class QuestionRequest(BaseModel):
|
| 14 |
question: str
|
| 15 |
-
|
|
|
|
|
|
|
| 16 |
class AnswerResponse(BaseModel):
|
| 17 |
-
sql: str
|
| 18 |
result: Any
|
| 19 |
-
|
|
|
|
|
|
|
| 20 |
@app.post("/")
|
| 21 |
def ask_question(req: QuestionRequest):
|
| 22 |
sql, result = mojica_bot.consult(req.question)
|
| 23 |
|
| 24 |
# Si es dataframe lo convertimos a json
|
| 25 |
-
if isinstance(result, pd.DataFrame):
|
| 26 |
result = result.to_dict(orient="records")
|
| 27 |
|
| 28 |
return {"sql": sql, "result": result}
|
| 29 |
# return {"sql": "WASA"}
|
| 30 |
|
|
|
|
| 31 |
# @app.post("/", response_model=AnswerResponse)
|
| 32 |
-
# def ask_question(req: QuestionRequest):
|
| 33 |
# sql, result = mojica_bot.consult(req.question)
|
| 34 |
-
|
| 35 |
# # * Si es dataframe lo convertimos a json
|
| 36 |
-
# if isinstance(result, pd.DataFrame):
|
| 37 |
# result = result.to_dict(orient="records")
|
| 38 |
-
|
| 39 |
# return {"sql": sql, "result": result}
|
| 40 |
|
| 41 |
# @app.get("/")
|
|
|
|
| 9 |
|
| 10 |
mojica_bot = MojicaAgent(Config)
|
| 11 |
|
| 12 |
+
|
| 13 |
# * Esquema de entrada como marshmellow
|
| 14 |
+
class QuestionRequest(BaseModel):
|
| 15 |
question: str
|
| 16 |
+
number: str
|
| 17 |
+
|
| 18 |
+
|
| 19 |
class AnswerResponse(BaseModel):
|
| 20 |
+
sql: str
|
| 21 |
result: Any
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
|
| 25 |
@app.post("/")
|
| 26 |
def ask_question(req: QuestionRequest):
|
| 27 |
sql, result = mojica_bot.consult(req.question)
|
| 28 |
|
| 29 |
# Si es dataframe lo convertimos a json
|
| 30 |
+
if isinstance(result, pd.DataFrame):
|
| 31 |
result = result.to_dict(orient="records")
|
| 32 |
|
| 33 |
return {"sql": sql, "result": result}
|
| 34 |
# return {"sql": "WASA"}
|
| 35 |
|
| 36 |
+
|
| 37 |
# @app.post("/", response_model=AnswerResponse)
|
| 38 |
+
# def ask_question(req: QuestionRequest):
|
| 39 |
# sql, result = mojica_bot.consult(req.question)
|
| 40 |
+
|
| 41 |
# # * Si es dataframe lo convertimos a json
|
| 42 |
+
# if isinstance(result, pd.DataFrame):
|
| 43 |
# result = result.to_dict(orient="records")
|
| 44 |
+
|
| 45 |
# return {"sql": sql, "result": result}
|
| 46 |
|
| 47 |
# @app.get("/")
|
config.py
CHANGED
|
@@ -5,11 +5,12 @@ from huggingface_hub import hf_hub_download
|
|
| 5 |
class Config:
|
| 6 |
DB_PATH = '/tmp/dataset.db'
|
| 7 |
TABLE_NAME = 'mojica_Ventas'
|
|
|
|
| 8 |
MODEL_NAME = "ibm-granite/granite-3b-code-instruct-128k"
|
| 9 |
CSV_PATH = "/kaggle/input/mojica-hoja-1/mojica_hoja_1.csv"
|
| 10 |
MAX_HISTORY = 3 # Mantener las últimas 3 interacciones (memoria)
|
| 11 |
-
MAX_TOKENS =
|
| 12 |
-
MAX_NEW_TOKENS =
|
| 13 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 14 |
|
| 15 |
if not os.path.exists(Config.DB_PATH):
|
|
|
|
| 5 |
class Config:
|
| 6 |
DB_PATH = '/tmp/dataset.db'
|
| 7 |
TABLE_NAME = 'mojica_Ventas'
|
| 8 |
+
EXMAPLES_JSON = '/examples.json'
|
| 9 |
MODEL_NAME = "ibm-granite/granite-3b-code-instruct-128k"
|
| 10 |
CSV_PATH = "/kaggle/input/mojica-hoja-1/mojica_hoja_1.csv"
|
| 11 |
MAX_HISTORY = 3 # Mantener las últimas 3 interacciones (memoria)
|
| 12 |
+
MAX_TOKENS = 4096
|
| 13 |
+
MAX_NEW_TOKENS = 256
|
| 14 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 15 |
|
| 16 |
if not os.path.exists(Config.DB_PATH):
|
examples.json
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"PRODUCTOS": [
|
| 3 |
+
{
|
| 4 |
+
"pregunta": "Top 5 productos más vendidos este año",
|
| 5 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND strftime('%Y', \"Fecha\") = '2025'\nGROUP BY \"Descripcion\"\nORDER BY total_vendido DESC\nLIMIT 5;"
|
| 6 |
+
},
|
| 7 |
+
{
|
| 8 |
+
"pregunta": "Productos con mayor margen de ganancia",
|
| 9 |
+
"query": "SELECT \"Descripcion\", (SUM(\"Neto\") / SUM(\"Cantidad\")) AS margen_unitario\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Cantidad\" > 0\nGROUP BY \"Descripcion\"\nHAVING SUM(\"Cantidad\") > 30\nORDER BY margen_unitario DESC\nLIMIT 10;"
|
| 10 |
+
},
|
| 11 |
+
{
|
| 12 |
+
"pregunta": "Productos con menor rotación en inventario",
|
| 13 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Fecha\" BETWEEN DATE('now','-6 months') AND DATE('now')\nGROUP BY \"Descripcion\"\nORDER BY total_vendido ASC\nLIMIT 5;"
|
| 14 |
+
}
|
| 15 |
+
],
|
| 16 |
+
"CLIENTES_CERO": [
|
| 17 |
+
{
|
| 18 |
+
"pregunta": "Total de clientes cero actuales",
|
| 19 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n);"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"pregunta": "Clientes cero que fueron VIP anteriormente",
|
| 23 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS historial_compra\nFROM \"sells\"\nWHERE \"Cliente\" IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n )\n) AND \"Cliente\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING SUM(\"Neto\") > 5000\nORDER BY historial_compra DESC;"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"pregunta": "Clientes cero por antigüedad de inactividad",
|
| 27 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", MAX(DATE(\"Fecha\")) AS ultima_compra,\n(julianday('now') - julianday(MAX(\"Fecha\"))) AS dias_inactivo\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n)\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY dias_inactivo DESC\nLIMIT 10;"
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"CIUDAD_CERO": [
|
| 31 |
+
{
|
| 32 |
+
"pregunta": "Clientes cero por ciudad",
|
| 33 |
+
"query": "SELECT \"Ciudad\", COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n)\nGROUP BY \"Ciudad\"\nORDER BY clientes_cero DESC;"
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"pregunta": "Ciudad con mayor porcentaje de clientes cero",
|
| 37 |
+
"query": "SELECT \"Ciudad\", \n(COUNT(DISTINCT CASE WHEN \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n) THEN \"Cliente\" END) * 100.0 / COUNT(DISTINCT \"Cliente\")) AS porcentaje_cero\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY porcentaje_cero DESC\nLIMIT 5;"
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"pregunta": "Clientes cero en Guadalajara con historial de compras",
|
| 41 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS historial_compra\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now', '-28 day')\n)\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING SUM(\"Neto\") > 1000\nORDER BY historial_compra DESC;"
|
| 42 |
+
}
|
| 43 |
+
],
|
| 44 |
+
"PRODUCTO_CIUDAD": [
|
| 45 |
+
{
|
| 46 |
+
"pregunta": "Producto más vendido por ciudad",
|
| 47 |
+
"query": "SELECT \"Ciudad\", \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL AND \"Descripcion\" IS NOT NULL\nGROUP BY \"Ciudad\", \"Descripcion\"\nORDER BY \"Ciudad\", total_vendido DESC;"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"pregunta": "Ciudades donde el queso gouda es más popular",
|
| 51 |
+
"query": "SELECT \"Ciudad\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Descripcion\" LIKE '%Queso Gouda%' AND \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY total_vendido DESC\nLIMIT 5;"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"pregunta": "Productos exclusivos por ciudad",
|
| 55 |
+
"query": "SELECT \"Ciudad\", \"Descripcion\"\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\", \"Descripcion\"\nHAVING COUNT(DISTINCT \"Ciudad\") = 1\nORDER BY \"Ciudad\";"
|
| 56 |
+
}
|
| 57 |
+
],
|
| 58 |
+
"CLIENTE_CIUDAD": [
|
| 59 |
+
{
|
| 60 |
+
"pregunta": "Clientes más valiosos por ciudad",
|
| 61 |
+
"query": "SELECT \"Ciudad\", \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS valor_total\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL AND \"Cliente\" IS NOT NULL\nGROUP BY \"Ciudad\", \"Cliente\", \"Razon Social\"\nORDER BY \"Ciudad\", valor_total DESC;"
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"pregunta": "Ciudad con clientes más frecuentes",
|
| 65 |
+
"query": "SELECT \"Ciudad\", AVG(compras_por_cliente) AS frecuencia_promedio\nFROM (\n SELECT \"Ciudad\", \"Cliente\", COUNT(*) AS compras_por_cliente\n FROM \"sells\"\n WHERE \"Ciudad\" IS NOT NULL AND \"Cliente\" IS NOT NULL\n GROUP BY \"Ciudad\", \"Cliente\"\n)\nGROUP BY \"Ciudad\"\nORDER BY frecuencia_promedio DESC;"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"pregunta": "Distribución de clientes por ciudad",
|
| 69 |
+
"query": "SELECT \"Ciudad\", COUNT(DISTINCT \"Cliente\") AS total_clientes\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL AND \"Cliente\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY total_clientes DESC;"
|
| 70 |
+
}
|
| 71 |
+
],
|
| 72 |
+
"PRODUCTO_DINERO": [
|
| 73 |
+
{
|
| 74 |
+
"pregunta": "Productos que generan más ingresos",
|
| 75 |
+
"query": "SELECT \"Descripcion\", SUM(\"Neto\") AS ingresos_totales\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY \"Descripcion\"\nORDER BY ingresos_totales DESC\nLIMIT 10;"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"pregunta": "Rentabilidad por categoría de producto",
|
| 79 |
+
"query": "SELECT \nCASE \n WHEN \"Descripcion\" LIKE '%queso%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%pan%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%bebida%' THEN 'Bebidas'\n ELSE 'Otros'\nEND AS categoria,\nSUM(\"Neto\") AS ingresos_totales\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY categoria\nORDER BY ingresos_totales DESC;"
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"pregunta": "Productos con mejor margen por unidad",
|
| 83 |
+
"query": "SELECT \"Descripcion\", (SUM(\"Neto\") / SUM(\"Cantidad\")) AS margen_unitario\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL AND \"Cantidad\" > 0\nGROUP BY \"Descripcion\"\nHAVING SUM(\"Cantidad\") > 20\nORDER BY margen_unitario DESC\nLIMIT 10;"
|
| 84 |
+
}
|
| 85 |
+
],
|
| 86 |
+
"CIUDAD_DINERO": [
|
| 87 |
+
{
|
| 88 |
+
"pregunta": "Ciudades con mayor facturación",
|
| 89 |
+
"query": "SELECT \"Ciudad\", SUM(\"Neto\") AS ingresos_totales\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY ingresos_totales DESC\nLIMIT 5;"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"pregunta": "Crecimiento de ventas por ciudad",
|
| 93 |
+
"query": "SELECT \"Ciudad\", \n(SUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = '2025-06' THEN \"Neto\" ELSE 0 END) - \nSUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = '2025-05' THEN \"Neto\" ELSE 0 END)) AS crecimiento_mensual\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY crecimiento_mensual DESC;"
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"pregunta": "Ticket promedio por ciudad",
|
| 97 |
+
"query": "SELECT \"Ciudad\", AVG(\"Neto\") AS ticket_promedio\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY ticket_promedio DESC;"
|
| 98 |
+
}
|
| 99 |
+
],
|
| 100 |
+
"CLIENTE_DINERO": [
|
| 101 |
+
{
|
| 102 |
+
"pregunta": "Top 10 clientes por valor de compras",
|
| 103 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS valor_total\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY valor_total DESC\nLIMIT 10;"
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"pregunta": "Clientes con mayor frecuencia de compra de alto valor",
|
| 107 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", COUNT(*) AS compras, AVG(\"Neto\") AS ticket_promedio\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL AND \"Neto\" > 500\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY compras DESC\nLIMIT 10;"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"pregunta": "Clientes con mayor potencial de crecimiento",
|
| 111 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", \n(MAX(\"Neto\") - AVG(\"Neto\")) AS potencial_crecimiento\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING COUNT(*) > 3\nORDER BY potencial_crecimiento DESC\nLIMIT 10;"
|
| 112 |
+
}
|
| 113 |
+
],
|
| 114 |
+
"TIEMPO_DINERO": [
|
| 115 |
+
{
|
| 116 |
+
"pregunta": "Ventas mensuales del año actual",
|
| 117 |
+
"query": "SELECT strftime('%Y-%m', \"Fecha\") AS mes, SUM(\"Neto\") AS ventas_mensuales\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL AND strftime('%Y', \"Fecha\") = '2025'\nGROUP BY mes\nORDER BY mes;"
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"pregunta": "Crecimiento interanual de ventas",
|
| 121 |
+
"query": "SELECT \nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2025' THEN \"Neto\" ELSE 0 END) AS ventas_2025,\nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2024' THEN \"Neto\" ELSE 0 END) AS ventas_2024,\n((SUM(CASE WHEN strftime('%Y', \"Fecha\") = '2025' THEN \"Neto\" ELSE 0 END) - \nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2024' THEN \"Neto\" ELSE 0 END)) * 100.0 / \nSUM(CASE WHEN strftime('%Y', \"Fecha\") = '2024' THEN \"Neto\" ELSE 0 END)) AS crecimiento_porcentual\nFROM \"sells\";"
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"pregunta": "Ventas por día de la semana",
|
| 125 |
+
"query": "SELECT \nCASE strftime('%w', \"Fecha\")\n WHEN '0' THEN 'Domingo'\n WHEN '1' THEN 'Lunes'\n WHEN '2' THEN 'Martes'\n WHEN '3' THEN 'Miércoles'\n WHEN '4' THEN 'Jueves'\n WHEN '5' THEN 'Viernes'\n WHEN '6' THEN 'Sábado'\nEND AS dia_semana,\nSUM(\"Neto\") AS ventas_dia\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL\nGROUP BY dia_semana\nORDER BY ventas_dia DESC;"
|
| 126 |
+
}
|
| 127 |
+
],
|
| 128 |
+
"CIUDADES": [
|
| 129 |
+
{
|
| 130 |
+
"pregunta": "¿Cuántas ciudades diferentes tenemos registradas?",
|
| 131 |
+
"query": "SELECT COUNT(DISTINCT \"Ciudad\") AS total_ciudades\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL;"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"pregunta": "Lista de las 10 ciudades con más transacciones",
|
| 135 |
+
"query": "SELECT \"Ciudad\", COUNT(*) AS total_transacciones\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nGROUP BY \"Ciudad\"\nORDER BY total_transacciones DESC\nLIMIT 10;"
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"pregunta": "Nombra 15 ciudades al azar de nuestra base",
|
| 139 |
+
"query": "SELECT DISTINCT \"Ciudad\"\nFROM \"sells\"\nWHERE \"Ciudad\" IS NOT NULL\nORDER BY RANDOM()\nLIMIT 15;"
|
| 140 |
+
}
|
| 141 |
+
],
|
| 142 |
+
"CLIENTES": [
|
| 143 |
+
{
|
| 144 |
+
"pregunta": "¿Cuántos clientes únicos tenemos registrados?",
|
| 145 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS total_clientes\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL;"
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"pregunta": "Muestra 10 clientes al azar con su razón social",
|
| 149 |
+
"query": "SELECT DISTINCT \"Cliente\", \"Razon Social\"\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nORDER BY RANDOM()\nLIMIT 10;"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"pregunta": "Clientes con nombres más largos en la base",
|
| 153 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", LENGTH(\"Razon Social\") AS longitud_nombre\nFROM \"sells\"\nWHERE \"Razon Social\" IS NOT NULL\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY longitud_nombre DESC\nLIMIT 10;"
|
| 154 |
+
}
|
| 155 |
+
],
|
| 156 |
+
"FECHAS": [
|
| 157 |
+
{
|
| 158 |
+
"pregunta": "¿Desde cuándo tenemos registros de ventas?",
|
| 159 |
+
"query": "SELECT MIN(DATE(\"Fecha\")) AS primera_fecha, MAX(DATE(\"Fecha\")) AS ultima_fecha\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL;"
|
| 160 |
+
},
|
| 161 |
+
{
|
| 162 |
+
"pregunta": "¿Cuántos días diferentes con ventas tenemos?",
|
| 163 |
+
"query": "SELECT COUNT(DISTINCT DATE(\"Fecha\")) AS dias_con_ventas\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL;"
|
| 164 |
+
},
|
| 165 |
+
{
|
| 166 |
+
"pregunta": "Rango de fechas de nuestro historial",
|
| 167 |
+
"query": "SELECT MIN(\"Fecha\") AS inicio_historial, MAX(\"Fecha\") AS fin_historial,\nJULIANDAY(MAX(\"Fecha\")) - JULIANDAY(MIN(\"Fecha\")) AS dias_totales\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL;"
|
| 168 |
+
}
|
| 169 |
+
],
|
| 170 |
+
"VOLUMEN_VENTAS": [
|
| 171 |
+
{
|
| 172 |
+
"pregunta": "¿Cuántas transacciones totales tenemos?",
|
| 173 |
+
"query": "SELECT COUNT(*) AS total_transacciones\nFROM \"sells\";"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"pregunta": "Promedio de ventas por día",
|
| 177 |
+
"query": "SELECT AVG(ventas_dia) AS promedio_ventas_diarias\nFROM (\n SELECT DATE(\"Fecha\"), SUM(\"Neto\") AS ventas_dia\n FROM \"sells\"\n WHERE \"Fecha\" IS NOT NULL\n GROUP BY DATE(\"Fecha\")\n);"
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"pregunta": "Día con mayor número de transacciones",
|
| 181 |
+
"query": "SELECT DATE(\"Fecha\") AS dia, COUNT(*) AS total_transacciones\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL\nGROUP BY dia\nORDER BY total_transacciones DESC\nLIMIT 1;"
|
| 182 |
+
}
|
| 183 |
+
],
|
| 184 |
+
"ESTADISTICAS_BASICAS": [
|
| 185 |
+
{
|
| 186 |
+
"pregunta": "Valor promedio de una venta",
|
| 187 |
+
"query": "SELECT AVG(\"Neto\") AS valor_promedio_venta\nFROM \"sells\"\nWHERE \"Neto\" IS NOT NULL;"
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"pregunta": "Cantidad promedio de productos por venta",
|
| 191 |
+
"query": "SELECT AVG(\"Cantidad\") AS cantidad_promedio\nFROM \"sells\"\nWHERE \"Cantidad\" IS NOT NULL;"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"pregunta": "Valor máximo y mínimo de una venta",
|
| 195 |
+
"query": "SELECT MAX(\"Neto\") AS venta_maxima, MIN(\"Neto\") AS venta_minima\nFROM \"sells\"\nWHERE \"Neto\" IS NOT NULL;"
|
| 196 |
+
}
|
| 197 |
+
],
|
| 198 |
+
"CATEGORIAS_PRODUCTOS": [
|
| 199 |
+
{
|
| 200 |
+
"pregunta": "¿Cuáles categorías de productos tenemos?",
|
| 201 |
+
"query": "SELECT DISTINCT \nCASE \n WHEN \"Descripcion\" LIKE '%QUESO%' OR \"Descripcion\" LIKE '%QUESO%' THEN 'Quesos'\n WHEN \"Descripcion\" LIKE '%LECHE%' OR \"Descripcion\" LIKE '%LACTEO%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%PAN%' OR \"Descripcion\" LIKE '%BOLLERIA%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%BEBIDA%' OR \"Descripcion\" LIKE '%REFRESCO%' THEN 'Bebidas'\n WHEN \"Descripcion\" LIKE '%EMBUTIDO%' OR \"Descripcion\" LIKE '%SALCHICHA%' THEN 'Embutidos'\n ELSE 'Otros'\nEND AS categoria\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nORDER BY categoria;"
|
| 202 |
+
},
|
| 203 |
+
{
|
| 204 |
+
"pregunta": "Cantidad de productos por categoría",
|
| 205 |
+
"query": "SELECT \nCASE \n WHEN \"Descripcion\" LIKE '%QUESO%' OR \"Descripcion\" LIKE '%QUESO%' THEN 'Quesos'\n WHEN \"Descripcion\" LIKE '%LECHE%' OR \"Descripcion\" LIKE '%LACTEO%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%PAN%' OR \"Descripcion\" LIKE '%BOLLERIA%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%BEBIDA%' OR \"Descripcion\" LIKE '%REFRESCO%' THEN 'Bebidas'\n WHEN \"Descripcion\" LIKE '%EMBUTIDO%' OR \"Descripcion\" LIKE '%SALCHICHA%' THEN 'Embutidos'\n ELSE 'Otros'\nEND AS categoria,\nCOUNT(DISTINCT \"Descripcion\") AS cantidad_productos\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY categoria\nORDER BY cantidad_productos DESC;"
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"pregunta": "Categorías con mayor variedad de productos",
|
| 209 |
+
"query": "SELECT \nCASE \n WHEN \"Descripcion\" LIKE '%QUESO%' OR \"Descripcion\" LIKE '%QUESO%' THEN 'Quesos'\n WHEN \"Descripcion\" LIKE '%LECHE%' OR \"Descripcion\" LIKE '%LACTEO%' THEN 'Lácteos'\n WHEN \"Descripcion\" LIKE '%PAN%' OR \"Descripcion\" LIKE '%BOLLERIA%' THEN 'Panadería'\n WHEN \"Descripcion\" LIKE '%BEBIDA%' OR \"Descripcion\" LIKE '%REFRESCO%' THEN 'Bebidas'\n WHEN \"Descripcion\" LIKE '%EMBUTIDO%' OR \"Descripcion\" LIKE '%SALCHICHA%' THEN 'Embutidos'\n ELSE 'Otros'\nEND AS categoria,\nCOUNT(DISTINCT \"Descripcion\") AS variedad_productos\nFROM \"sells\"\nWHERE \"Descripcion\" IS NOT NULL\nGROUP BY categoria\nORDER BY variedad_productos DESC\nLIMIT 3;"
|
| 210 |
+
}
|
| 211 |
+
],
|
| 212 |
+
"CLIENTES_CERO_TIEMPO": [
|
| 213 |
+
{
|
| 214 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Marzo 2025?",
|
| 215 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-03-31')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-03-31','-27 day') AND DATE('2025-03-31')\n);"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en el primer trimestre de 2025?",
|
| 219 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-03-31')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-03-31','-27 day') AND DATE('2025-03-31')\n);"
|
| 220 |
+
},
|
| 221 |
+
{
|
| 222 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en el último semestre?",
|
| 223 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('now','-6 months')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('now','-6 months','-27 day') AND DATE('now','-6 months')\n);"
|
| 224 |
+
}
|
| 225 |
+
],
|
| 226 |
+
"CIUDAD_CERO_TIEMPO": [
|
| 227 |
+
{
|
| 228 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Guadalajara en Marzo 2025?",
|
| 229 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-03-31')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-03-31','-27 day') AND DATE('2025-03-31')\n);"
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Monterrey en Abril 2025?",
|
| 233 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('2025-04-30')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('2025-04-30','-27 day') AND DATE('2025-04-30')\n);"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"pregunta": "¿Cuántos clientes cero tuvimos en Zapopan en el último mes?",
|
| 237 |
+
"query": "SELECT COUNT(DISTINCT \"Cliente\") AS clientes_cero\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Zapopan%'\nAND \"Cliente\" IS NOT NULL\nAND DATE(\"Fecha\") <= DATE('now','-1 month')\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") BETWEEN DATE('now','-1 month','-27 day') AND DATE('now','-1 month')\n);"
|
| 238 |
+
}
|
| 239 |
+
],
|
| 240 |
+
"PRODUCTO_CIUDAD_TIEMPO": [
|
| 241 |
+
{
|
| 242 |
+
"pregunta": "¿Qué productos se vendieron más en Guadalajara en Marzo?",
|
| 243 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Descripcion\" IS NOT NULL\nAND strftime('%Y-%m', \"Fecha\") = '2025-03'\nGROUP BY \"Descripcion\"\nORDER BY total_vendido DESC\nLIMIT 5;"
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"pregunta": "¿Cuál fue el producto más vendido en Monterrey en Abril?",
|
| 247 |
+
"query": "SELECT \"Descripcion\", SUM(\"Cantidad\") AS total_vendido\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Descripcion\" IS NOT NULL\nAND strftime('%Y-%m', \"Fecha\") = '2025-04'\nGROUP BY \"Descripcion\"\nORDER BY total_vendido DESC\nLIMIT 1;"
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"pregunta": "Productos con mayor crecimiento en Zapopan el último trimestre",
|
| 251 |
+
"query": "SELECT \"Descripcion\", \n(SUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = strftime('%Y-%m','now') THEN \"Cantidad\" ELSE 0 END) - \nSUM(CASE WHEN strftime('%Y-%m', \"Fecha\") = strftime('%Y-%m','now','-3 months') THEN \"Cantidad\" ELSE 0 END)) AS crecimiento\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Zapopan%'\nAND \"Descripcion\" IS NOT NULL\nGROUP BY \"Descripcion\"\nORDER BY crecimiento DESC\nLIMIT 5;"
|
| 252 |
+
}
|
| 253 |
+
],
|
| 254 |
+
"CLIENTE_CIUDAD_TIEMPO": [
|
| 255 |
+
{
|
| 256 |
+
"pregunta": "Clientes más valiosos de Guadalajara en el último trimestre",
|
| 257 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", SUM(\"Neto\") AS valor_total\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Cliente\" IS NOT NULL\nAND \"Fecha\" BETWEEN DATE('now','-3 months') AND DATE('now')\nGROUP BY \"Cliente\", \"Razon Social\"\nORDER BY valor_total DESC\nLIMIT 10;"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"pregunta": "Clientes nuevos en Monterrey durante Marzo",
|
| 261 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", MIN(DATE(\"Fecha\")) AS primera_compra\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Cliente\" IS NOT NULL\nAND strftime('%Y-%m', \"Fecha\") = '2025-03'\nGROUP BY \"Cliente\", \"Razon Social\"\nHAVING MIN(DATE(\"Fecha\")) >= DATE('2025-03-01')\nORDER BY primera_compra;"
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"pregunta": "Clientes inactivos en Zapopan que compraron hace 6 meses",
|
| 265 |
+
"query": "SELECT \"Cliente\", \"Razon Social\", MAX(DATE(\"Fecha\")) AS ultima_compra\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Zapopan%'\nAND \"Cliente\" IS NOT NULL\nAND \"Cliente\" NOT IN (\n SELECT DISTINCT \"Cliente\"\n FROM \"sells\"\n WHERE DATE(\"Fecha\") > DATE('now','-28 day')\n)\nAND DATE(\"Fecha\") BETWEEN DATE('now','-6 months') AND DATE('now','-5 months')\nGROUP BY \"Cliente\", \"Razon Social\";"
|
| 266 |
+
}
|
| 267 |
+
],
|
| 268 |
+
"TIEMPO_DINERO_CIUDAD": [
|
| 269 |
+
{
|
| 270 |
+
"pregunta": "Ventas totales por mes en Guadalajara",
|
| 271 |
+
"query": "SELECT strftime('%Y-%m', \"Fecha\") AS mes, SUM(\"Neto\") AS ventas_mensuales\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Guadalajara%'\nAND \"Fecha\" IS NOT NULL\nGROUP BY mes\nORDER BY mes DESC;"
|
| 272 |
+
},
|
| 273 |
+
{
|
| 274 |
+
"pregunta": "Crecimiento de ventas en Monterrey por trimestre",
|
| 275 |
+
"query": "SELECT \nCASE \n WHEN strftime('%m', \"Fecha\") BETWEEN '01' AND '03' THEN 'Q1'\n WHEN strftime('%m', \"Fecha\") BETWEEN '04' AND '06' THEN 'Q2'\n WHEN strftime('%m', \"Fecha\") BETWEEN '07' AND '09' THEN 'Q3'\n WHEN strftime('%m', \"Fecha\") BETWEEN '10' AND '12' THEN 'Q4'\nEND AS trimestre,\nSUM(\"Neto\") AS ventas_trimestrales\nFROM \"sells\"\nWHERE \"Ciudad\" LIKE '%Monterrey%'\nAND \"Fecha\" IS NOT NULL\nGROUP BY trimestre\nORDER BY trimestre;"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"pregunta": "Comparativo de ventas: Guadalajara vs Monterrey por mes",
|
| 279 |
+
"query": "SELECT strftime('%Y-%m', \"Fecha\") AS mes,\nSUM(CASE WHEN \"Ciudad\" LIKE '%Guadalajara%' THEN \"Neto\" ELSE 0 END) AS ventas_guadalajara,\nSUM(CASE WHEN \"Ciudad\" LIKE '%Monterrey%' THEN \"Neto\" ELSE 0 END) AS ventas_monterrey\nFROM \"sells\"\nWHERE \"Fecha\" IS NOT NULL\nGROUP BY mes\nORDER BY mes DESC;"
|
| 280 |
+
}
|
| 281 |
+
]
|
| 282 |
+
}
|
intelligent_question_router.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from supervised_classifier import QuestionClassifier
|
| 2 |
+
import torch
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
class IntelligentQuestionRouter:
|
| 6 |
+
def __init__(self):
|
| 7 |
+
self.classifier = QuestionClassifier()
|
| 8 |
+
self._initialize_json()
|
| 9 |
+
|
| 10 |
+
def _initialize_json(self):
|
| 11 |
+
with open("examples.json", "r", encoding="utf-8") as f:
|
| 12 |
+
self.examples = json.load(f)
|
| 13 |
+
|
| 14 |
+
def _get_examples(self, category):
|
| 15 |
+
return self.examples.get(category, self.examples["CLIENTES_CERO"])
|
| 16 |
+
|
| 17 |
+
def route_question(self, question: str):
|
| 18 |
+
try:
|
| 19 |
+
ml_category_id = self.classifier.predict(question)
|
| 20 |
+
return self._get_examples(ml_category_id)
|
| 21 |
+
except Exception as e:
|
| 22 |
+
print("Error in routing question:", e)
|
| 23 |
+
return self._get_examples("CLIENTES_CERO")
|
| 24 |
+
|
| 25 |
+
|
load_json.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
def load_examples(path: str = 'examples.json'):
|
| 3 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 4 |
+
return json.load(f)
|
| 5 |
+
|
| 6 |
+
examples = []
|
| 7 |
+
for category, items in data.items():
|
| 8 |
+
for item in items:
|
| 9 |
+
examples.append({
|
| 10 |
+
"category": category,
|
| 11 |
+
"question": item["pregunta"],
|
| 12 |
+
"sql": item["sql"]
|
| 13 |
+
})
|
| 14 |
+
return examples
|
memory.py
CHANGED
|
@@ -1,47 +1,26 @@
|
|
| 1 |
-
from
|
| 2 |
-
|
| 3 |
-
from
|
| 4 |
|
| 5 |
-
class ConversationMemory:
|
| 6 |
-
def __init__(self, max_history: int = Config.MAX_HISTORY):
|
| 7 |
-
self.history = deque(maxlen=max_history)
|
| 8 |
-
self.schema_cache = None
|
| 9 |
-
|
| 10 |
-
def add_interaction(self, question: str, sql: str, result: str):
|
| 11 |
-
self.history.append({
|
| 12 |
-
"question": question,
|
| 13 |
-
"sql": sql,
|
| 14 |
-
"result_summary": self._summarize_result(result)
|
| 15 |
-
})
|
| 16 |
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
else:
|
| 27 |
-
return f"Filas: {len(result)}, Columnas: {list(result.columns)}"
|
| 28 |
-
return str(result)
|
| 29 |
-
|
| 30 |
-
def get_context(self, current_question: str) -> str:
|
| 31 |
-
if not self.history:
|
| 32 |
-
return ""
|
| 33 |
-
last_relevant = []
|
| 34 |
-
for interaction in self.history:
|
| 35 |
-
if "producto" in interaction['question'].lower() and "producto" in current_question.lower():
|
| 36 |
-
last_relevant.append(interaction)
|
| 37 |
-
elif "cliente" in interaction['question'].lower() and "cliente" in current_question.lower():
|
| 38 |
-
last_relevant.append(interaction)
|
| 39 |
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from langchain.vectorstores import Chroma
|
| 2 |
+
from langchain.embeddings import HuggingFaceEmbeddings
|
| 3 |
+
from langchain_core.documents import Document
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
+
class Memory:
|
| 7 |
+
def __init__(self):
|
| 8 |
+
self.embedding_model = HuggingFaceEmbeddings(
|
| 9 |
+
model_name="sentence-transformers/all-MiniLM-L6-v2"
|
| 10 |
+
)
|
| 11 |
+
self.vector_store = Chroma(
|
| 12 |
+
persist_directory="./chroma_db", embedding_function=self.embedding_model
|
| 13 |
+
)
|
| 14 |
+
self.schema_cache = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
def add_interaction(self, question: str, answer: str, sql: str):
|
| 17 |
+
document = Document(
|
| 18 |
+
page_content=f"Pregunta: {question}\nRespuesta: {answer}\nSQL: {sql}",
|
| 19 |
+
metadata={"source": "interaction"},
|
| 20 |
+
)
|
| 21 |
+
self.vector_store.add_documents([document])
|
| 22 |
+
|
| 23 |
+
def get_relevant_interactions(self, question: str, top_k=3):
|
| 24 |
+
results = self.vector_store.similarity_search(question, k=top_k)
|
| 25 |
+
return "\n".join([d.page_content for d in results])
|
| 26 |
+
|
mojica_agent.py
CHANGED
|
@@ -1,18 +1,21 @@
|
|
| 1 |
-
from memory import
|
| 2 |
from config import Config
|
| 3 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
| 4 |
import torch, gc
|
| 5 |
import unicodedata
|
| 6 |
-
from typing import Dict, Tuple, Optional
|
| 7 |
import re
|
| 8 |
import pandas as pd
|
| 9 |
import sqlite3
|
|
|
|
| 10 |
|
| 11 |
|
| 12 |
class MojicaAgent:
|
| 13 |
def __init__(self, config: Config):
|
| 14 |
self.config = config
|
| 15 |
-
self.memory =
|
|
|
|
|
|
|
| 16 |
self.essential_columns = [
|
| 17 |
{
|
| 18 |
"name": "Descripcion",
|
|
@@ -34,172 +37,111 @@ class MojicaAgent:
|
|
| 34 |
},
|
| 35 |
{"name": "Neto", "type": "REAL", "description": "Valor neto de la venta"},
|
| 36 |
]
|
| 37 |
-
self.
|
| 38 |
-
self.
|
| 39 |
|
| 40 |
-
def
|
| 41 |
def try_load_model():
|
| 42 |
-
tokenizer = AutoTokenizer.from_pretrained(self.config.MODEL_NAME)
|
| 43 |
-
model = AutoModelForCausalLM.from_pretrained(
|
| 44 |
self.config.MODEL_NAME,
|
| 45 |
-
|
| 46 |
-
device_map="auto", # * <- Distribuye mejor entre GPU
|
| 47 |
torch_dtype="auto",
|
|
|
|
| 48 |
).eval()
|
| 49 |
-
return tokenizer, model
|
| 50 |
|
| 51 |
try:
|
| 52 |
-
|
| 53 |
except torch.cuda.OutOfMemoryError:
|
| 54 |
-
# * Libera memoria e intenta de nuevo
|
| 55 |
gc.collect()
|
| 56 |
torch.cuda.empty_cache()
|
| 57 |
torch.cuda.ipc_collect()
|
| 58 |
-
|
| 59 |
|
| 60 |
-
def
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
{"
|
| 66 |
]
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
|
|
|
| 70 |
|
| 71 |
-
def
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
return f"""
|
| 76 |
-
SELECT DISTINCT "Cliente", "Razon Social"
|
| 77 |
-
FROM "{table}"
|
| 78 |
-
WHERE "Cliente" IS NOT NULL
|
| 79 |
-
AND "Cliente" NOT IN (
|
| 80 |
-
SELECT DISTINCT "Cliente"
|
| 81 |
-
FROM "{table}"
|
| 82 |
-
WHERE DATE("Fecha") >= DATE('now', '-28 day')
|
| 83 |
-
);
|
| 84 |
-
"""
|
| 85 |
-
return None
|
| 86 |
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
key = candidate.lower()
|
| 91 |
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
if corrected:
|
| 96 |
-
return f'"{corrected}"'
|
| 97 |
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
if corrected:
|
| 101 |
-
return f'"{corrected}"'
|
| 102 |
|
| 103 |
-
|
|
|
|
| 104 |
|
| 105 |
-
|
| 106 |
-
cursor
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
"Sales": "sells",
|
| 117 |
-
}
|
| 118 |
-
# * Opcional (convierte la key en minuscula solo si el valor esta en real_columns)
|
| 119 |
-
alias_map = {
|
| 120 |
-
key.lower(): value
|
| 121 |
-
for key, value in aliases.items()
|
| 122 |
-
if value in real_columns
|
| 123 |
-
}
|
| 124 |
-
pattern = r'"(\w+)"'
|
| 125 |
|
| 126 |
-
|
| 127 |
-
|
|
|
|
|
|
|
| 128 |
|
| 129 |
-
def
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
table_name = self.schema["table_name"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
|
| 133 |
-
# * Detectamos el tipo de pregunta para asignar los ejemplos
|
| 134 |
-
question_type = (
|
| 135 |
-
"PRODUCTOS"
|
| 136 |
-
if "producto" in question.lower()
|
| 137 |
-
else "CLIENTES" if "cliente" in question.lower() else "GENERAL"
|
| 138 |
-
)
|
| 139 |
-
examples = {
|
| 140 |
-
"PRODUCTOS": (
|
| 141 |
-
"-- P: 'Top 10 productos más vendidos'\n"
|
| 142 |
-
'SELECT "Descripcion", SUM("Cantidad") AS total_vendido\n'
|
| 143 |
-
f'FROM "{table_name}"\n'
|
| 144 |
-
'WHERE "Descripcion" IS NOT NULL\n'
|
| 145 |
-
'GROUP BY "Descripcion"\n'
|
| 146 |
-
"ORDER BY total_vendido DESC\n"
|
| 147 |
-
"LIMIT 10;\n\n"
|
| 148 |
-
"-- P: 'Productos con mayor valor neto'\n"
|
| 149 |
-
'SELECT "Descripcion", SUM("Neto") AS valor_total\n'
|
| 150 |
-
f'FROM "{table_name}"\n'
|
| 151 |
-
'WHERE "Descripcion" IS NOT NULL\n'
|
| 152 |
-
'GROUP BY "Descripcion"\n'
|
| 153 |
-
"ORDER BY valor_total DESC\n"
|
| 154 |
-
"LIMIT 5;"
|
| 155 |
-
),
|
| 156 |
-
"CLIENTES": (
|
| 157 |
-
"-- P: 'Top 5 clientes con mayor valor neto'\n"
|
| 158 |
-
'SELECT "Cliente", SUM("Neto") AS valor_total\n'
|
| 159 |
-
f'FROM "{table_name}"\n'
|
| 160 |
-
"WHERE \"Cliente\" IS NOT NULL AND \"Fecha\" BETWEEN '2025-01-01' AND '2025-12-31'\n"
|
| 161 |
-
'GROUP BY "Cliente"\n'
|
| 162 |
-
"ORDER BY valor_total DESC\n"
|
| 163 |
-
"LIMIT 5;\n\n"
|
| 164 |
-
"-- P: 'Clientes con más compras en marzo'\n"
|
| 165 |
-
'SELECT "Cliente", COUNT(*) AS total_compras\n'
|
| 166 |
-
f'FROM "{table_name}"\n'
|
| 167 |
-
"WHERE \"Cliente\" IS NOT NULL AND strftime('%m', \"Fecha\") = '03'\n"
|
| 168 |
-
'GROUP BY "Cliente"\n'
|
| 169 |
-
"ORDER BY total_compras DESC\n"
|
| 170 |
-
"LIMIT 10;\n\n"
|
| 171 |
-
"-- P: 'Clientes de Guadalajara con más compras'\n"
|
| 172 |
-
'SELECT "Cliente", "Razon Social", COUNT(*) AS total_compras\n'
|
| 173 |
-
f'FROM "{table_name}"\n'
|
| 174 |
-
'WHERE "Cliente" IS NOT NULL AND "Ciudad" = \'Guadalajara\'\n'
|
| 175 |
-
'GROUP BY "Cliente", "Razon Social"\n'
|
| 176 |
-
"ORDER BY total_compras DESC\n"
|
| 177 |
-
"LIMIT 10;"
|
| 178 |
-
),
|
| 179 |
-
"GENERAL": (
|
| 180 |
-
"-- P: 'Ventas totales por mes'\n"
|
| 181 |
-
'SELECT strftime(\'%m\', "Fecha") AS mes, SUM("Neto") AS ventas\n'
|
| 182 |
-
f'FROM "{table_name}"\n'
|
| 183 |
-
"WHERE mes IS NOT NULL\n"
|
| 184 |
-
"GROUP BY mes\n"
|
| 185 |
-
"ORDER BY mes;\n\n"
|
| 186 |
-
"-- P: 'Producto menos vendido en 2025'\n"
|
| 187 |
-
'SELECT "Descripcion", SUM("Cantidad") AS total_vendido\n'
|
| 188 |
-
f'FROM "{table_name}"\n'
|
| 189 |
-
"WHERE \"Descripcion\" IS NOT NULL AND \"Fecha\" BETWEEN '2025-01-01' AND '2025-12-31'\n"
|
| 190 |
-
'GROUP BY "Descripcion"\n'
|
| 191 |
-
"ORDER BY total_vendido ASC\n"
|
| 192 |
-
"LIMIT 1;"
|
| 193 |
-
),
|
| 194 |
-
}
|
| 195 |
-
# * Retornamos el prompt
|
| 196 |
return (
|
| 197 |
f"""
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
+ "\n".join(
|
| 204 |
[
|
| 205 |
f"- {col['name']} ({col['type']}): {col['description']}"
|
|
@@ -207,114 +149,161 @@ class MojicaAgent:
|
|
| 207 |
]
|
| 208 |
)
|
| 209 |
+ f"""
|
| 210 |
-
|
| 211 |
-
### CONTEXTO (Últimas interacciones) ###
|
| 212 |
-
{memory_context if memory_context else "Sin historial relevante"}
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
### EJEMPLOS ({question_type}) ###
|
| 216 |
-
{examples[question_type]}
|
| 217 |
-
|
| 218 |
-
### REGLAS CRÍTICAS ###
|
| 219 |
-
- Usar siempre nombres exactos de columnas
|
| 220 |
-
- Usar solo las columnas listadas
|
| 221 |
-
- Prohibido inventar columnas
|
| 222 |
-
- Para el nombre del cliente, usar SIEMPRE "Razon Social".
|
| 223 |
-
- Para un mes específico usar: strftime('%m', "Fecha") = 'MM'
|
| 224 |
-
- Para cantidades usar SUM("Cantidad"), para dinero usar SUM("Neto")
|
| 225 |
-
- Para ciudad usar SIEMPRE "Ciudad"
|
| 226 |
-
- Agrupar por la dimensión principal (producto/cliente)
|
| 227 |
-
- Ordenar DESC para 'más/mayor', ASC para 'menos/menor'
|
| 228 |
-
- Usar LIMIT para top N
|
| 229 |
-
- Contesta siempre en el idioma en el que se te pregunta no traduzcas.
|
| 230 |
-
- Año actual: 2025
|
| 231 |
-
- Siempre terminar con un LIMIT = 1 en caso que se indique lo contrario
|
| 232 |
-
- Para 'más vendido' usar SUM("Cantidad"), para 'mayor valor' usar SUM("Neto")
|
| 233 |
-
- Usar "Razon Social" cuando pregunten por el nombre del cliente
|
| 234 |
-
- Usar "Ciudad" para filtrar o agrupar por ubicación
|
| 235 |
-
- Queda estrictamente prohibido usar acentos
|
| 236 |
-
- **Siempre excluir valores nulos con 'IS NOT NULL' en las columnas usadas en WHERE, GROUP BY u ORDER BY**
|
| 237 |
-
|
| 238 |
-
### PREGUNTA ACTUAL ###
|
| 239 |
-
\"\"\"{question}\"\"\"
|
| 240 |
|
| 241 |
-
|
| 242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
)
|
| 244 |
|
| 245 |
-
def
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
|
| 254 |
-
|
|
|
|
| 255 |
sql_matches = list(
|
| 256 |
re.finditer(
|
| 257 |
r"(SELECT|WITH|INSERT|UPDATE|DELETE)[\s\S]+?;", output, re.IGNORECASE
|
| 258 |
)
|
| 259 |
)
|
| 260 |
|
| 261 |
-
# * si no existe nada entonces retornamos None
|
| 262 |
if not sql_matches:
|
| 263 |
return None
|
| 264 |
|
| 265 |
-
#
|
| 266 |
-
sql = sql_matches[-1].group(0).strip()
|
| 267 |
|
| 268 |
-
#
|
| 269 |
if any(
|
| 270 |
-
|
| 271 |
-
for
|
| 272 |
):
|
| 273 |
return None
|
| 274 |
-
|
|
|
|
| 275 |
if not sql.endswith(";"):
|
| 276 |
sql += ";"
|
| 277 |
|
| 278 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
sql = remove_accents(sql)
|
| 280 |
|
| 281 |
-
#
|
| 282 |
-
|
| 283 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
|
| 285 |
-
# * SQL limpio mas no corregido de erores, corregumos los errores y despues enviamos
|
| 286 |
validate_sql = self._validate_and_correct_sql(sql)
|
| 287 |
return validate_sql
|
| 288 |
|
| 289 |
-
def
|
| 290 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
try:
|
| 292 |
-
|
| 293 |
-
connection.close()
|
| 294 |
-
return result
|
| 295 |
except Exception as e:
|
| 296 |
-
return f"Error
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
def consult(self, question: str) -> Tuple[str, any]:
|
| 301 |
-
if self._handle_speecial_cases(question=question) != None:
|
| 302 |
-
sql_query = self._handle_speecial_cases(question=question)
|
| 303 |
-
result = self._execute_sql(sql_query)
|
| 304 |
-
# self.memory.add_interaction(question, sql_query, result)
|
| 305 |
-
return sql_query, result
|
| 306 |
-
|
| 307 |
-
prompt = self._build_prompt(question)
|
| 308 |
tokenized_input = self.tokenizer(
|
| 309 |
-
|
| 310 |
return_tensors="pt",
|
| 311 |
truncation=True,
|
| 312 |
max_length=self.config.MAX_TOKENS,
|
| 313 |
).to(self.config.DEVICE)
|
| 314 |
-
|
| 315 |
-
# * Desactiva el cálculo de gradientes -> Siempre poner cuando se haga prediccion
|
| 316 |
-
# * - Reduce consumo de memoria
|
| 317 |
-
# * - Acelera inferencia
|
| 318 |
with torch.no_grad():
|
| 319 |
tokenized_output_model = self.model.generate(
|
| 320 |
**tokenized_input,
|
|
@@ -326,17 +315,33 @@ class MojicaAgent:
|
|
| 326 |
do_sample=True,
|
| 327 |
pad_token_id=self.tokenizer.eos_token_id,
|
| 328 |
)
|
| 329 |
-
|
| 330 |
output_model = self.tokenizer.decode(
|
| 331 |
tokenized_output_model[0], skip_special_tokens=True
|
| 332 |
)
|
|
|
|
| 333 |
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from memory import Memory as VectorMemory
|
| 2 |
from config import Config
|
| 3 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
| 4 |
import torch, gc
|
| 5 |
import unicodedata
|
| 6 |
+
from typing import Dict, Tuple, Optional, Any
|
| 7 |
import re
|
| 8 |
import pandas as pd
|
| 9 |
import sqlite3
|
| 10 |
+
from intelligent_question_router import IntelligentQuestionRouter
|
| 11 |
|
| 12 |
|
| 13 |
class MojicaAgent:
|
| 14 |
def __init__(self, config: Config):
|
| 15 |
self.config = config
|
| 16 |
+
self.memory = VectorMemory()
|
| 17 |
+
self.router = IntelligentQuestionRouter()
|
| 18 |
+
self._load_training_data()
|
| 19 |
self.essential_columns = [
|
| 20 |
{
|
| 21 |
"name": "Descripcion",
|
|
|
|
| 37 |
},
|
| 38 |
{"name": "Neto", "type": "REAL", "description": "Valor neto de la venta"},
|
| 39 |
]
|
| 40 |
+
self._initialize_database()
|
| 41 |
+
self._initialize_model()
|
| 42 |
|
| 43 |
+
def _initialize_model(self):
|
| 44 |
def try_load_model():
|
| 45 |
+
self.tokenizer = AutoTokenizer.from_pretrained(self.config.MODEL_NAME)
|
| 46 |
+
self.model = AutoModelForCausalLM.from_pretrained(
|
| 47 |
self.config.MODEL_NAME,
|
| 48 |
+
device_map="auto",
|
|
|
|
| 49 |
torch_dtype="auto",
|
| 50 |
+
trust_remote_code=True,
|
| 51 |
).eval()
|
|
|
|
| 52 |
|
| 53 |
try:
|
| 54 |
+
try_load_model()
|
| 55 |
except torch.cuda.OutOfMemoryError:
|
|
|
|
| 56 |
gc.collect()
|
| 57 |
torch.cuda.empty_cache()
|
| 58 |
torch.cuda.ipc_collect()
|
| 59 |
+
try_load_model()
|
| 60 |
|
| 61 |
+
def _load_training_data(self):
|
| 62 |
+
training_examples = [
|
| 63 |
+
{"question": "productos más vendidos", "category": "producto"},
|
| 64 |
+
{"question": "mejor producto", "category": "producto"},
|
| 65 |
+
{"question": "clientes que más compran", "category": "cliente"},
|
| 66 |
+
{"question": "clientes inactivos", "category": "cliente"},
|
| 67 |
]
|
| 68 |
+
try:
|
| 69 |
+
self.router.semantic_classifier.train(training_examples)
|
| 70 |
+
except Exception as e:
|
| 71 |
+
print(f"Error training semantic classifier: {e}")
|
| 72 |
|
| 73 |
+
def _validate_result_existing(self, result):
|
| 74 |
+
# Si es un string de error
|
| 75 |
+
if isinstance(result, str) and "Error" in result:
|
| 76 |
+
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
+
# Si es un DataFrame vacío
|
| 79 |
+
if hasattr(result, "empty") and result.empty:
|
| 80 |
+
return False
|
|
|
|
| 81 |
|
| 82 |
+
# Si es una lista vacía
|
| 83 |
+
if isinstance(result, list) and len(result) == 0:
|
| 84 |
+
return False
|
|
|
|
|
|
|
| 85 |
|
| 86 |
+
# En cualquier otro caso, asumimos éxito
|
| 87 |
+
return True
|
|
|
|
|
|
|
| 88 |
|
| 89 |
+
def _initialize_database(self):
|
| 90 |
+
self.conn = sqlite3.connect(self.config.DB_PATH)
|
| 91 |
|
| 92 |
+
cursor = self.conn.cursor()
|
| 93 |
+
cursor.execute(f"DROP TABLE IF EXISTS {self.config.TABLE_NAME}")
|
| 94 |
+
self.conn.commit()
|
| 95 |
+
df = pd.read_csv(self.config.CSV_PATH, low_memory=False)
|
| 96 |
+
|
| 97 |
+
real_cols = [
|
| 98 |
+
col["name"] for col in self.essential_columns if col["type"] == "REAL"
|
| 99 |
+
]
|
| 100 |
+
for col in real_cols:
|
| 101 |
+
if col in df.columns:
|
| 102 |
+
df[col] = pd.to_numeric(df[col], errors="coerce")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
+
df.to_sql(self.config.TABLE_NAME, self.conn, if_exists="replace", index=False)
|
| 105 |
+
self.schema = self._get_schema_structured()
|
| 106 |
+
# Configuracion de pandas:
|
| 107 |
+
pd.set_option("display.float_format", "{:,.2f}".format)
|
| 108 |
|
| 109 |
+
def _get_schema_structured(self) -> Dict:
|
| 110 |
+
if self.memory.schema_cache:
|
| 111 |
+
return self.memory.schema_cache
|
| 112 |
+
cursor = self.conn.cursor()
|
| 113 |
+
cursor.execute(f"PRAGMA table_info({self.config.TABLE_NAME})")
|
| 114 |
+
columns = [
|
| 115 |
+
{"name": column[1], "type": column[2]} for column in cursor.fetchall()
|
| 116 |
+
]
|
| 117 |
+
schema = {"table_name": self.config.TABLE_NAME, "columns": columns}
|
| 118 |
+
self.memory.schema_cache = schema
|
| 119 |
+
return schema
|
| 120 |
+
|
| 121 |
+
def _generate_sql_prompt(self, question: str) -> str:
|
| 122 |
+
memory_context = self.memory.get_relevant_memory(question)
|
| 123 |
table_name = self.schema["table_name"]
|
| 124 |
+
# Uso del router
|
| 125 |
+
try:
|
| 126 |
+
examples_list = self.router.route_question(question)
|
| 127 |
+
# Convertir ejemplos a texto para el prompt
|
| 128 |
+
examples_text = "\n".join(
|
| 129 |
+
[f"-- P: '{ex['pregunta']}'\n{ex['query']}\n" for ex in examples_list]
|
| 130 |
+
)
|
| 131 |
+
question_type = "ROUTED_EXAMPLES"
|
| 132 |
+
except Exception as e:
|
| 133 |
+
print(f"Router failed, using manual detection: {e}")
|
| 134 |
+
# Fallback a detección manual
|
| 135 |
+
# question_type = self._detect_question_type_manual(question)
|
| 136 |
+
# examples_text = self.examples.get(question_type, "")
|
| 137 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
return (
|
| 139 |
f"""
|
| 140 |
+
### TAREA ###
|
| 141 |
+
Generar SOLO código SQL para la pregunta, usando EXCLUSIVAMENTE la tabla: "{table_name}"
|
| 142 |
+
|
| 143 |
+
### COLUMNAS RELEVANTES ###
|
| 144 |
+
"""
|
| 145 |
+ "\n".join(
|
| 146 |
[
|
| 147 |
f"- {col['name']} ({col['type']}): {col['description']}"
|
|
|
|
| 149 |
]
|
| 150 |
)
|
| 151 |
+ f"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
|
| 153 |
+
### CONTEXTO (Últimas interacciones) ###
|
| 154 |
+
{memory_context if memory_context else "Sin historial relevante"}
|
| 155 |
+
|
| 156 |
+
### EJEMPLOS ###
|
| 157 |
+
{examples_text}
|
| 158 |
+
|
| 159 |
+
### REGLAS CRÍTICAS ###
|
| 160 |
+
- Usar siempre nombres exactos de columnas
|
| 161 |
+
- Usar solo las columnas listadas
|
| 162 |
+
- Prohibido inventar columnas
|
| 163 |
+
- Para el nombre del cliente, usar SIEMPRE "Razon Social".
|
| 164 |
+
- Para un mes específico usar: strftime('%m', "Fecha") = 'MM'
|
| 165 |
+
- Para cantidades usar SUM("Cantidad"), para dinero usar SUM("Neto")
|
| 166 |
+
- Agrupar por la dimensión principal (producto/cliente)
|
| 167 |
+
- Ordenar DESC para 'más/mayor', ASC para 'menos/menor'
|
| 168 |
+
- Contesta siempre en el idioma en el que se te pregunta no traduzcas.
|
| 169 |
+
- Año actual: 2025
|
| 170 |
+
- No inventes columnas o tablas que no existan
|
| 171 |
+
- Para preguntas sobre clientes cero, SIEMPRE usar la subconsulta NOT IN con las últimas 4 semanas.
|
| 172 |
+
- Si se menciona una ciudad, incluir el filtro AND "Ciudad" LIKE '%...%'
|
| 173 |
+
- Usa LIMIT cuando se te pida un numero finito de datos
|
| 174 |
+
- Para 'más vendido' usar SUM("Cantidad"), para 'mayor valor' usar SUM("Neto")
|
| 175 |
+
- Usar "Razon Social" cuando pregunten por el nombre del cliente
|
| 176 |
+
- Usar "Ciudad" para filtrar o agrupar por ubicación
|
| 177 |
+
- Queda estrictamente prohibido usar acentos
|
| 178 |
+
- **Siempre excluir valores nulos con 'IS NOT NULL' en las columnas usadas en WHERE, GROUP BY u ORDER BY**
|
| 179 |
+
- Para preguntas sobre ciudad SIEMPRE incluir "Ciudad" en la query
|
| 180 |
+
- Para busquedas por Descripcion siempre usar LIKE
|
| 181 |
+
- Mandar solo la cantidad de rows que el usuario pide.
|
| 182 |
+
### PREGUNTA ACTUAL ###
|
| 183 |
+
\"\"\"{question}\"\"\"
|
| 184 |
+
|
| 185 |
+
### SQL:
|
| 186 |
+
"""
|
| 187 |
)
|
| 188 |
|
| 189 |
+
def _generate_analysis_prompt(self, question: str, result: Any) -> str:
|
| 190 |
+
return f"""
|
| 191 |
+
Basado EXCLUSIVAMENTE en estos datos: {result}
|
| 192 |
+
|
| 193 |
+
Responde esta pregunta: {question}
|
| 194 |
+
|
| 195 |
+
Reglas estrictas:
|
| 196 |
+
- Nunca inventes numeros
|
| 197 |
+
- Usa solo datos proporcionados
|
| 198 |
+
- Maximo una oracion
|
| 199 |
+
"""
|
| 200 |
+
|
| 201 |
+
def _clean_analysis_output(self, ouput: str) -> Optional[str]:
|
| 202 |
+
pattern = r"Respuesta:([\s\S]+)"
|
| 203 |
+
match = re.search(pattern, ouput)
|
| 204 |
+
if match:
|
| 205 |
+
return match.group(1).strip()
|
| 206 |
+
else:
|
| 207 |
+
return "Sin análisis"
|
| 208 |
|
| 209 |
+
def _clean_sql_output(self, output: str) -> Optional[str]:
|
| 210 |
+
# Encuentra todas las posibles queries completas que terminen en ;
|
| 211 |
sql_matches = list(
|
| 212 |
re.finditer(
|
| 213 |
r"(SELECT|WITH|INSERT|UPDATE|DELETE)[\s\S]+?;", output, re.IGNORECASE
|
| 214 |
)
|
| 215 |
)
|
| 216 |
|
|
|
|
| 217 |
if not sql_matches:
|
| 218 |
return None
|
| 219 |
|
| 220 |
+
# Tomar la última query encontrada
|
| 221 |
+
sql = sql_matches[-1].group(0).strip()
|
| 222 |
|
| 223 |
+
# Seguridad: bloquear queries peligrosas
|
| 224 |
if any(
|
| 225 |
+
cmd in sql.upper()
|
| 226 |
+
for cmd in ["DROP", "DELETE", "UPDATE", "INSERT", "ALTER"]
|
| 227 |
):
|
| 228 |
return None
|
| 229 |
+
|
| 230 |
+
# Asegurar que termine en ;
|
| 231 |
if not sql.endswith(";"):
|
| 232 |
sql += ";"
|
| 233 |
|
| 234 |
+
# ────────────────────────────────
|
| 235 |
+
# 1. Quitar acentos de toda la query
|
| 236 |
+
# ────────────────────────────────
|
| 237 |
+
def remove_accents(text: str) -> str:
|
| 238 |
+
return "".join(
|
| 239 |
+
c
|
| 240 |
+
for c in unicodedata.normalize("NFKD", text)
|
| 241 |
+
if not unicodedata.combining(c)
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
sql = remove_accents(sql)
|
| 245 |
|
| 246 |
+
# ────────────────────────────────
|
| 247 |
+
# 2. Agregar LIMIT si no existe
|
| 248 |
+
# ────────────────────────────────
|
| 249 |
+
# Buscar si ya hay un LIMIT en la query
|
| 250 |
+
# if not re.search(r"\bLIMIT\s+\d+", sql, re.IGNORECASE):
|
| 251 |
+
# # Insertar antes del último punto y coma
|
| 252 |
+
# sql = sql[:-1] + " LIMIT 1;" # puedes cambiar 100 por el valor default que quieras
|
| 253 |
|
|
|
|
| 254 |
validate_sql = self._validate_and_correct_sql(sql)
|
| 255 |
return validate_sql
|
| 256 |
|
| 257 |
+
def _validate_and_correct_sql(self, sql: str) -> str:
|
| 258 |
+
cur = self.conn.cursor()
|
| 259 |
+
cur.execute(f'PRAGMA table_info("{self.config.TABLE_NAME}")')
|
| 260 |
+
real_columns = [row[1] for row in cur.fetchall()]
|
| 261 |
+
column_lower_map = {col.lower(): col for col in real_columns}
|
| 262 |
+
aliases = {
|
| 263 |
+
"city": "Ciudad",
|
| 264 |
+
"client": "Cliente",
|
| 265 |
+
"razon_social": "Razon Social",
|
| 266 |
+
"razón social": "Razon Social",
|
| 267 |
+
"Sales": "sells",
|
| 268 |
+
'"Date"': "Fecha",
|
| 269 |
+
"mojica_Clientes": "sells",
|
| 270 |
+
"value_total": "valor_total",
|
| 271 |
+
"strstrftime": "strftime",
|
| 272 |
+
}
|
| 273 |
+
alias_map = {k.lower(): v for k, v in aliases.items()}
|
| 274 |
+
|
| 275 |
+
pattern = r"\b\w+\b"
|
| 276 |
+
|
| 277 |
+
def replace_column(m):
|
| 278 |
+
candidate = m.group(0) # Palabra encontrada
|
| 279 |
+
key = candidate.lower()
|
| 280 |
+
# ¿Es una columna?
|
| 281 |
+
corrected = column_lower_map.get(key)
|
| 282 |
+
if corrected:
|
| 283 |
+
return corrected
|
| 284 |
+
|
| 285 |
+
# ¿Es una alias?
|
| 286 |
+
corrected = alias_map.get(key)
|
| 287 |
+
if corrected is not None:
|
| 288 |
+
return corrected
|
| 289 |
+
return candidate # si no encuentra nada, lo deja igual
|
| 290 |
+
|
| 291 |
+
return re.sub(pattern, replace_column, sql).replace("\\", "")
|
| 292 |
+
|
| 293 |
+
def _execute_sql(self, sql: str) -> Any:
|
| 294 |
try:
|
| 295 |
+
return pd.read_sql_query(sql, self.conn)
|
|
|
|
|
|
|
| 296 |
except Exception as e:
|
| 297 |
+
return f"Error: {str(e)}"
|
| 298 |
+
|
| 299 |
+
def consult(self, question: str) -> Tuple[str, Any, str]:
|
| 300 |
+
sql_prompt = self._generate_sql_prompt(question)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
tokenized_input = self.tokenizer(
|
| 302 |
+
sql_prompt,
|
| 303 |
return_tensors="pt",
|
| 304 |
truncation=True,
|
| 305 |
max_length=self.config.MAX_TOKENS,
|
| 306 |
).to(self.config.DEVICE)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
with torch.no_grad():
|
| 308 |
tokenized_output_model = self.model.generate(
|
| 309 |
**tokenized_input,
|
|
|
|
| 315 |
do_sample=True,
|
| 316 |
pad_token_id=self.tokenizer.eos_token_id,
|
| 317 |
)
|
|
|
|
| 318 |
output_model = self.tokenizer.decode(
|
| 319 |
tokenized_output_model[0], skip_special_tokens=True
|
| 320 |
)
|
| 321 |
+
sql = self._clean_sql_output(output_model)
|
| 322 |
|
| 323 |
+
# * Ejecución de SQL y generación de analisis
|
| 324 |
+
result = self._execute_sql(sql)
|
| 325 |
+
# * INICIO DE ANALISIS (COMENTADO)
|
| 326 |
+
# Analisis
|
| 327 |
+
# analysis_prompt = self._generate_analysis_prompt(question, result)
|
| 328 |
+
# analyzed_token_input = self.tokenizer(
|
| 329 |
+
# analysis_prompt,
|
| 330 |
+
# return_tensors="pt",
|
| 331 |
+
# truncation=True,
|
| 332 |
+
# max_length=self.config.MAX_TOKENS,
|
| 333 |
+
# ).to(self.config.DEVICE)
|
| 334 |
+
# with torch.no_grad():
|
| 335 |
+
# tokenized_analysis_output_model = self.model.generate(
|
| 336 |
+
# **analyzed_token_input,
|
| 337 |
+
# max_new_tokens=self.config.MAX_NEW_TOKENS,
|
| 338 |
+
# temperature=0.65,
|
| 339 |
+
# )
|
| 340 |
+
# analysis = self.tokenizer.decode(
|
| 341 |
+
# tokenized_analysis_output_model[0], skip_special_tokens=True
|
| 342 |
+
# )
|
| 343 |
+
# analysis = self._clean_analysis_output(analysis)
|
| 344 |
+
# analysis <- LE quite ese parametro
|
| 345 |
+
# * FIN DE ANALISIS (COMENTADO)
|
| 346 |
+
self.memory.add_interaction(question=question, answer=result, sql=sql)
|
| 347 |
+
return sql, result
|
requirements.txt
CHANGED
|
Binary files a/requirements.txt and b/requirements.txt differ
|
|
|
semantic_classifier.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from sentence_transformers import SentenceTransformer
|
| 2 |
+
from sklearn.cluster import KMeans
|
| 3 |
+
from config import Config
|
| 4 |
+
from load_json import load_examples
|
| 5 |
+
|
| 6 |
+
from sentence_transformers import SentenceTransformer
|
| 7 |
+
from sklearn.cluster import KMeans
|
| 8 |
+
|
| 9 |
+
class SemanticClassifier:
|
| 10 |
+
def __init__(self, model_name="paraphrase-multilingual-MiniLM-L12-v2", initialized_train=True):
|
| 11 |
+
self.model = SentenceTransformer(model_name)
|
| 12 |
+
self.clusters = {}
|
| 13 |
+
self.examples_embeddings = None
|
| 14 |
+
self.kmeans = None
|
| 15 |
+
if initialized_train:
|
| 16 |
+
self.train()
|
| 17 |
+
|
| 18 |
+
def train(self, train_data=Config.EXMAPLES_JSON, n_clusters=15):
|
| 19 |
+
examples = load_examples(train_data)
|
| 20 |
+
|
| 21 |
+
# * Aplanar ejemplos
|
| 22 |
+
flat_examples = []
|
| 23 |
+
for category, items in examples.items():
|
| 24 |
+
for item in items:
|
| 25 |
+
flat_examples.append({
|
| 26 |
+
"category": category,
|
| 27 |
+
"pregunta": item["pregunta"],
|
| 28 |
+
"query": item["query"]
|
| 29 |
+
})
|
| 30 |
+
|
| 31 |
+
questions = [ex["pregunta"] for ex in flat_examples]
|
| 32 |
+
|
| 33 |
+
# * Obtener embeddings
|
| 34 |
+
embeddings = self.model.encode(questions)
|
| 35 |
+
|
| 36 |
+
# * Clustering
|
| 37 |
+
self.kmeans = KMeans(n_clusters=n_clusters, random_state=12)
|
| 38 |
+
cluster_ids = self.kmeans.fit_predict(embeddings)
|
| 39 |
+
|
| 40 |
+
# * Guardar ejemplos por cluster
|
| 41 |
+
for i, cluster_id in enumerate(cluster_ids):
|
| 42 |
+
# * Crear lista si no existe
|
| 43 |
+
if cluster_id not in self.clusters:
|
| 44 |
+
self.clusters[cluster_id] = []
|
| 45 |
+
# * Agregamos el ejemplo
|
| 46 |
+
self.clusters[cluster_id].append(flat_examples[i])
|
| 47 |
+
self.examples_embeddings = embeddings
|
| 48 |
+
|
| 49 |
+
def classify(self, question: str):
|
| 50 |
+
# * En formato de embedding
|
| 51 |
+
question_embedding = self.model.encode([question])
|
| 52 |
+
# * Encontrar el cluster más cercano
|
| 53 |
+
cluster_id = self.kmeans.predict(question_embedding)[0]
|
| 54 |
+
# * Retornamos los ejemplos de ese cluster
|
| 55 |
+
return self.clusters.get(cluster_id, [])
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
# * FORMA DE USARSE
|
| 59 |
+
# classifier = SemanticClassifier()
|
| 60 |
+
# classifier.train(Config.EXMAPLES_JSON, n_clusters=5)
|
| 61 |
+
|
| 62 |
+
# resultado = classifier.classify("¿Cuantas ciudades tenemos registradas?")
|
| 63 |
+
# print(resultado) # te devuelve ejemplos de ese cluster
|
supervised_classifier.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import (
|
| 2 |
+
AutoTokenizer,
|
| 3 |
+
AutoModelForSequenceClassification,
|
| 4 |
+
Trainer,
|
| 5 |
+
TrainingArguments,
|
| 6 |
+
)
|
| 7 |
+
from config import Config
|
| 8 |
+
import json
|
| 9 |
+
from datasets import Dataset
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class QuestionClassifier:
|
| 14 |
+
def __init__(
|
| 15 |
+
self, model_name="distilbert-base-multilingual-cased", initialized_train=True
|
| 16 |
+
):
|
| 17 |
+
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 18 |
+
self.model_name = model_name
|
| 19 |
+
self.category2id = None
|
| 20 |
+
self.category2id = None
|
| 21 |
+
if initialized_train:
|
| 22 |
+
self.train()
|
| 23 |
+
|
| 24 |
+
def train(self, json_path=Config.EXMAPLES_JSON, num_epochs=3):
|
| 25 |
+
# * Cargar ejemplos
|
| 26 |
+
with open(json_path, "r", encoding="utf-8") as f:
|
| 27 |
+
examples = json.load(f)
|
| 28 |
+
texts, labels, category2id = self._prepare_supervised_data(examples)
|
| 29 |
+
self.category2id = category2id
|
| 30 |
+
self.id2category = {value: key for key, value in category2id.items()}
|
| 31 |
+
self.model = AutoModelForSequenceClassification.from_pretrained(
|
| 32 |
+
self.model_name, num_labels=len(category2id)
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
encodings = self.tokenizer(texts, truncation=True, padding=True)
|
| 36 |
+
dataset = Dataset.from_dict(
|
| 37 |
+
{
|
| 38 |
+
"input_ids": encodings["input_ids"],
|
| 39 |
+
"attention_mask": encodings["attention_mask"],
|
| 40 |
+
"labels": labels,
|
| 41 |
+
}
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
training_args = TrainingArguments(
|
| 45 |
+
output_dir="./results",
|
| 46 |
+
per_device_train_batch_size=8,
|
| 47 |
+
num_train_epochs=num_epochs,
|
| 48 |
+
logging_steps=1,
|
| 49 |
+
# logging_strategy="steps",
|
| 50 |
+
report_to="none",
|
| 51 |
+
save_strategy="no",
|
| 52 |
+
remove_unused_columns=False,
|
| 53 |
+
eval_strategy="no",
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
# 4. Trainer
|
| 57 |
+
trainer = Trainer(model=self.model, args=training_args, train_dataset=dataset)
|
| 58 |
+
|
| 59 |
+
trainer.train()
|
| 60 |
+
|
| 61 |
+
def _prepare_supervised_data(self, examples):
|
| 62 |
+
category2id = {cat: i for i, cat in enumerate(examples.keys())}
|
| 63 |
+
texts = []
|
| 64 |
+
labels = []
|
| 65 |
+
for category, items in examples.items():
|
| 66 |
+
for item in items:
|
| 67 |
+
texts.append(item["pregunta"])
|
| 68 |
+
labels.append(category2id[category])
|
| 69 |
+
return texts, labels, category2id
|
| 70 |
+
|
| 71 |
+
def predict(self, question: str):
|
| 72 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 73 |
+
self.model.to(device)
|
| 74 |
+
|
| 75 |
+
inputs = self.tokenizer(
|
| 76 |
+
question, return_tensors="pt", truncation=True, padding=True
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
inputs = {k: v.to(device) for k, v in inputs.items()}
|
| 80 |
+
|
| 81 |
+
with torch.no_grad():
|
| 82 |
+
outputs = self.model(**inputs)
|
| 83 |
+
predicted_class_id = outputs.logits.argmax().item()
|
| 84 |
+
|
| 85 |
+
return self.id2category[predicted_class_id]
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# * FORMA DE USARSE
|
| 89 |
+
# qc = QuestionClassifier()
|
| 90 |
+
# qc.train()
|
| 91 |
+
|
| 92 |
+
# categoria = qc.predict("Dame los productos más vendidos")
|
| 93 |
+
# print(categoria) # → 'PRODUCTOS'
|