refactor

prep_viewer_data.py

@@ -7,6 +7,7 @@ from huggingface_hub import list_datasets
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
 
+
 # Initialize the HTTP client
 client = httpx.AsyncClient(timeout=60, http2=True)
 
@@ -115,6 +116,8 @@ async def prep_data(sample_size=200_000, min_likes=1):
     df = pl.read_parquet(
         "hf://datasets/davanstrien/dataset-viewer-descriptions-processed/data/train-00000-of-00001.parquet"
     )
+    # remove datasets that are already in the train or test set we can remove this later once the model works okay
+
     in_train_or_test = set(df["dataset_id"].unique().to_list())
 
     # Get all datasets