build a clean cache

app.py

@@ -1,5 +1,3 @@
-# app.py – Gradio Space wrapper for modular_graph_and_candidates
-
 from __future__ import annotations
 
 import json
@@ -9,36 +7,31 @@ import tempfile
 from datetime import datetime, timedelta
 from functools import lru_cache
 from pathlib import Path
+from huggingface_hub import hf_hub_download
 
 import gradio as gr
 
-# —— refactored helpers ——
 from modular_graph_and_candidates import build_graph_json, generate_html, build_timeline_json, generate_timeline_html
 
 HF_MAIN_REPO = "https://github.com/huggingface/transformers"
 
-# ───────────────────────────── cache repo once per 24 h ───────────────────────────
-
-@lru_cache(maxsize=4)
-def clone_or_cache(repo_url: str) -> Path:
-    """Shallow‑clone *repo_url* and reuse it for 24 h."""
-    tmp_root = Path(tempfile.gettempdir())
-    cache_dir = tmp_root / f"repo_{abs(hash(repo_url))}"
-    stamp = cache_dir / ".cloned_at"
-
-    if cache_dir.exists() and stamp.exists():
-        try:
-            if datetime.utcnow() - datetime.fromisoformat(stamp.read_text().strip()) < timedelta(days=1):
-                return cache_dir
-        except Exception:
-            pass  # fall through → reclone
-        shutil.rmtree(cache_dir, ignore_errors=True)
+CACHE_REPO = "Molbap/hf_cached_embeds_log"
 
-    subprocess.check_call(["git", "clone", "--depth", "1", repo_url, str(cache_dir)])
-    stamp.write_text(datetime.utcnow().isoformat())
-    return cache_dir
+def _fetch_from_cache_repo(kind: str, sim_method: str, threshold: float, multimodal: bool):
+    repo_id = CACHE_REPO
+    latest_fp = hf_hub_download(repo_id=repo_id, filename="latest.json", repo_type="dataset")
+    info = json.loads(Path(latest_fp).read_text(encoding="utf-8"))
+    sha = info.get("sha")
+    key = f"{sha}/{sim_method}-{threshold:.2f}-m{int(multimodal)}"
+    html_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.html", repo_type="dataset")
+    json_fp = hf_hub_download(repo_id=repo_id, filename=f"{kind}/{key}.json", repo_type="dataset")
+    raw_html = Path(html_fp).read_text(encoding="utf-8")
+    json_text = Path(json_fp).read_text(encoding="utf-8")
+    iframe_html = f'<iframe style="width:100%;height:85vh;border:none;" srcdoc="{_escape_srcdoc(raw_html)}"></iframe>'
+    tmp = Path(tempfile.mkstemp(suffix=("_timeline.json" if kind == "timeline" else ".json"))[1])
+    tmp.write_text(json_text, encoding="utf-8")
+    return iframe_html, str(tmp)
 
-# ───────────────────────────── main callback ─────────────────────────────────────
 
 def _escape_srcdoc(text: str) -> str:
     """Escape for inclusion inside an <iframe srcdoc="…"> attribute."""
@@ -52,48 +45,13 @@ def _escape_srcdoc(text: str) -> str:
 
 
 def run_graph(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
-    """Generate the dependency graph visualization."""
-    repo_path = clone_or_cache(repo_url)
-
-    graph = build_graph_json(
-        transformers_dir=repo_path,
-        threshold=threshold,
-        multimodal=multimodal,
-        sim_method=sim_method,
-    )
-
-    raw_html = generate_html(graph)
+    return _fetch_from_cache_repo("graph", sim_method, threshold, multimodal)
 
-    iframe_html = (
-        f'<iframe style="width:100%;height:85vh;border:none;" '
-        f'srcdoc="{_escape_srcdoc(raw_html)}"></iframe>'
-    )
-
-    tmp_json = Path(tempfile.mktemp(suffix=".json"))
-    tmp_json.write_text(json.dumps(graph), encoding="utf-8")
-    return iframe_html, str(tmp_json)
 
 def run_timeline(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
-    """Generate the chronological timeline visualization."""
-    repo_path = clone_or_cache(repo_url)
-
-    timeline = build_timeline_json(
-        transformers_dir=repo_path,
-        threshold=threshold,
-        multimodal=multimodal,
-        sim_method=sim_method,
-    )
+    return _fetch_from_cache_repo("timeline", sim_method, threshold, multimodal)
 
-    raw_html = generate_timeline_html(timeline)
-
-    iframe_html = (
-        f'<iframe style="width:100%;height:85vh;border:none;" '
-        f'srcdoc="{_escape_srcdoc(raw_html)}"></iframe>'
-    )
 
-    tmp_json = Path(tempfile.mktemp(suffix="_timeline.json"))
-    tmp_json.write_text(json.dumps(timeline), encoding="utf-8")
-    return iframe_html, str(tmp_json)
 
 # ───────────────────────────── UI ────────────────────────────────────────────────
 

build_cache.py

@@ -0,0 +1,64 @@
+# build_cache.py
+import os
+import io
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+from datetime import datetime, timezone
+from huggingface_hub import HfApi
+from modular_graph_and_candidates import (
+    build_graph_json, generate_html,
+    build_timeline_json, generate_timeline_html
+)
+
+REPO_URL   = os.getenv("REPO_URL", "https://github.com/huggingface/transformers")
+CACHE_REPO = "Molbap/hf_cached_embeds_log"
+THRESH     = float(os.getenv("SIM_THRESHOLD", "0.50"))
+MULTIMODAL = os.getenv("MULTIMODAL", "0") in {"1","true","True","YES","yes"}
+SIM_METHOD = os.getenv("SIM_METHOD", "jaccard")
+
+def main():
+    tmp = Path(tempfile.mkdtemp())
+    subprocess.check_call(["git", "clone", "--depth", "1", REPO_URL, str(tmp / "repo")])
+    sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=tmp / "repo", text=True).strip()
+    repo_path = tmp / "repo"
+
+    graph = build_graph_json(repo_path, threshold=THRESH, multimodal=MULTIMODAL, sim_method=SIM_METHOD)
+    timeline = build_timeline_json(repo_path, threshold=THRESH, multimodal=MULTIMODAL, sim_method=SIM_METHOD)
+    graph_html = generate_html(graph)
+    timeline_html = generate_timeline_html(timeline)
+
+    api = HfApi()
+    api.create_repo(repo_id=CACHE_REPO, repo_type="dataset", exist_ok=True)
+
+    key = f"{sha}/{SIM_METHOD}-{THRESH:.2f}-m{int(MULTIMODAL)}"
+    latest = {
+        "sha": sha,
+        "updated_utc": datetime.now(timezone.utc).isoformat(),
+        "defaults": {"sim_method": SIM_METHOD, "threshold": THRESH, "multimodal": MULTIMODAL},
+        "paths": {
+            "graph_json":    f"graph/{key}.json",
+            "graph_html":    f"graph/{key}.html",
+            "timeline_json": f"timeline/{key}.json",
+            "timeline_html": f"timeline/{key}.html",
+        },
+    }
+
+    def put(path_in_repo: str, text: str):
+        api.upload_file(
+            path_or_fileobj=io.BytesIO(text.encode("utf-8")),
+            path_in_repo=path_in_repo,
+            repo_id=CACHE_REPO,
+            repo_type="dataset",
+            commit_message=f"cache {path_in_repo}",
+        )
+
+    put(f"graph/{key}.json", json.dumps(graph, separators=(",", ":")))
+    put(f"graph/{key}.html", graph_html)
+    put(f"timeline/{key}.json", json.dumps(timeline, separators=(",", ":")))
+    put(f"timeline/{key}.html", timeline_html)
+    put("latest.json", json.dumps(latest, separators=(",", ":")))
+
+if __name__ == "__main__":
+    main()

schedule_job.json

@@ -0,0 +1,23 @@
+{
+  "jobSpec": {
+    "dockerImage": "python:3.12",
+    "command": [
+      "bash",
+      "-lc",
+      "apt-get update -y && apt-get install -y git ca-certificates && git clone --depth 1 https://github.com/Molbap/transformers-modular-refactor.git /w && pip install -U huggingface_hub numpy tqdm sentence-transformers torch spaces && python /w/build_cache.py"
+    ],
+    "flavor": "cpu-basic",
+    "timeout": 7200,
+    "env": {
+      "REPO_URL": "https://github.com/huggingface/transformers",
+      "SIM_THRESHOLD": "0.50",
+      "MULTIMODAL": "0",
+      "SIM_METHOD": "jaccard"
+    },
+    "secrets": ["HF_TOKEN"],
+    "arch": "amd64"
+  },
+  "schedule": "8 9 * * *",
+  "suspend": false,
+  "concurrency": false
+}