Narsil HF staff commited on
Commit
19316e6
1 Parent(s): e89bd1d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +20 -6
app.py CHANGED
@@ -3,18 +3,24 @@ import gradio as gr
3
  import datetime
4
  import tempfile
5
  from huggingface_hub import hf_hub_download
 
 
 
 
 
6
 
7
  def download_very_slow(repo_id):
8
  os.environ.pop("HF_TRANSFER", None)
9
  os.environ["HF_CHUNK_SIZE"] = "1024"
10
 
11
  with tempfile.TemporaryDirectory() as workdir:
12
- hf_hub_download(
13
  repo_id,
14
  filename="pytorch_model.bin",
15
  force_download=True,
16
  cache_dir=workdir,
17
  )
 
18
 
19
 
20
  def download_slow(repo_id):
@@ -22,42 +28,50 @@ def download_slow(repo_id):
22
  os.environ["HF_CHUNK_SIZE"] = "10485760"
23
 
24
  with tempfile.TemporaryDirectory() as workdir:
25
- hf_hub_download(
26
  repo_id,
27
  filename="pytorch_model.bin",
28
  force_download=True,
29
  cache_dir=workdir,
30
  )
 
31
 
32
 
33
  def download_fast(repo_id):
34
  os.environ["HF_TRANSFER"] = "1"
35
  with tempfile.TemporaryDirectory() as workdir:
36
- hf_hub_download(
37
  repo_id,
38
  filename="pytorch_model.bin",
39
  force_download=True,
40
  cache_dir=workdir,
41
  )
 
42
 
43
 
44
  def download(repo_id):
45
  start = datetime.datetime.now()
46
- download_very_slow(repo_id)
47
  taken_very_slow = datetime.datetime.now() - start
48
 
49
  start = datetime.datetime.now()
50
- download_slow(repo_id)
51
  taken_slow = datetime.datetime.now() - start
52
 
53
  start = datetime.datetime.now()
54
- download_fast(repo_id)
55
  taken_fast = datetime.datetime.now() - start
56
 
57
  return f"""
58
  Very slow (huggingface_hub previous to https://github.com/huggingface/huggingface_hub/pull/1267): {taken_very_slow}
 
 
59
  Slow (huggingface_hub after): {taken_slow}
 
 
60
  Fast (with hf_transfer): {taken_fast}
 
 
61
  """
62
 
63
  examples = ["gpt2", "openai/whisper-large-v2"]
 
3
  import datetime
4
  import tempfile
5
  from huggingface_hub import hf_hub_download
6
+ import subprocess
7
+
8
+ def md5(filename):
9
+ return subprocess.check_output(["md5sum", filename])
10
+
11
 
12
  def download_very_slow(repo_id):
13
  os.environ.pop("HF_TRANSFER", None)
14
  os.environ["HF_CHUNK_SIZE"] = "1024"
15
 
16
  with tempfile.TemporaryDirectory() as workdir:
17
+ filename = hf_hub_download(
18
  repo_id,
19
  filename="pytorch_model.bin",
20
  force_download=True,
21
  cache_dir=workdir,
22
  )
23
+ return md5(filename)
24
 
25
 
26
  def download_slow(repo_id):
 
28
  os.environ["HF_CHUNK_SIZE"] = "10485760"
29
 
30
  with tempfile.TemporaryDirectory() as workdir:
31
+ filename = hf_hub_download(
32
  repo_id,
33
  filename="pytorch_model.bin",
34
  force_download=True,
35
  cache_dir=workdir,
36
  )
37
+ return md5(filename)
38
 
39
 
40
  def download_fast(repo_id):
41
  os.environ["HF_TRANSFER"] = "1"
42
  with tempfile.TemporaryDirectory() as workdir:
43
+ filename = hf_hub_download(
44
  repo_id,
45
  filename="pytorch_model.bin",
46
  force_download=True,
47
  cache_dir=workdir,
48
  )
49
+ return md5(filename)
50
 
51
 
52
  def download(repo_id):
53
  start = datetime.datetime.now()
54
+ md5_very_slow = download_very_slow(repo_id)
55
  taken_very_slow = datetime.datetime.now() - start
56
 
57
  start = datetime.datetime.now()
58
+ md5_slow = download_slow(repo_id)
59
  taken_slow = datetime.datetime.now() - start
60
 
61
  start = datetime.datetime.now()
62
+ md5_fasdt = download_fast(repo_id)
63
  taken_fast = datetime.datetime.now() - start
64
 
65
  return f"""
66
  Very slow (huggingface_hub previous to https://github.com/huggingface/huggingface_hub/pull/1267): {taken_very_slow}
67
+ MD5: {md5_very_slow}
68
+
69
  Slow (huggingface_hub after): {taken_slow}
70
+ MD5: {md5_slow}
71
+
72
  Fast (with hf_transfer): {taken_fast}
73
+ MD5: {md5_fast}
74
+
75
  """
76
 
77
  examples = ["gpt2", "openai/whisper-large-v2"]