Update app.py

app.py

@@ -1,8 +1,11 @@
-from fastapi import FastAPI
+from tokenizers import models, trainers, Tokenizer
 
-# Δημιουργία FastAPI εφαρμογής
-app = FastAPI()
+tokenizer = Tokenizer(model=models.WordPiece(unk_token="[UNK]"))
 
-@app.get("/")
-async def root():
-    return {"message": "Welcome to your basic FastAPI application!"}
+special_tokens = ["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"]
+trainer = trainers.WordPieceTrainer(vocab_size=25000, special_tokens=special_tokens)
+
+tokenizer.train(["wikitext-2.txt"], trainer=trainer)
+
+encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences.")
+print(encoding.ids)