Refactor update_datasets

app.py

@@ -7,6 +7,24 @@ import huggingface_hub as hfh
 from apscheduler.schedulers.background import BackgroundScheduler
 
 
+DATASET_ID = "albertvillanova/datasets-report"
+DATASET_PATH = "dataset"
+DATA_DIR = "data"
+DATA_PATH = f"{DATASET_PATH}{DATA_DIR}"
+
+
+def pull_dataset_repo(repo_id=DATASET_ID, repo_path=DATASET_PATH):
+    token = os.environ.get('HUB_TOKEN')
+    repo = hfh.Repository(
+        local_dir=repo_path,
+        clone_from=repo_id,
+        repo_type="dataset",
+        use_auth_token=token,
+    )
+    repo.git_pull()
+    return repo
+
+
 def greet(name):
     return "Hello " + name + "!!"
 
@@ -16,17 +34,11 @@ iface = gr.Interface(fn=greet, inputs="text", outputs="text")
 
 
 def update_datasets():
+    # Retrieve datasets
     datasets = hfh.list_datasets()
-    #
-    token = os.environ.get('HUB_TOKEN')
-    repo = hfh.Repository(
-        local_dir="dataset",
-        clone_from="albertvillanova/datasets-report",
-        repo_type="dataset",
-        use_auth_token=token,
-    )
-    repo.git_pull()
-    os.makedirs("dataset/data", exist_ok=True)
+    # Save dataset IDs
+    repo = pull_dataset_repo()
+    os.makedirs(DATA_PATH, exist_ok=True)
     today = datetime.datetime.now(datetime.timezone.utc).date().isoformat()
     with repo.commit(f"Add {today} data file"):
         with open(f"data/{today}.json", "w") as f: