Answers.Body: <p>Not Python, but you can do this with <a href="https://biojulia.net/BioStructures.jl/stable" rel="nofollow noreferrer">BioStructures.jl</a> in Julia:</p>
  <pre><code>using BioStructures
  pdblist = pdbentrylist()
  for pdbid in pdblist
      # Download a file and delete it after use
      downloadpdb(pdbid, format=MMCIF) do fp
          s = read(fp, MMCIF)
          # Do something with the file, e.g. print the cysteine fraction
          nres = countresidues(s, standardselector)
          if nres &gt; 0
              frac = countresidues(s, standardselector, x -&gt; resname(x) == &quot;CYS&quot;) / nres
              println(pdbid, &quot;  &quot;, round(frac; digits=2))
          end
      end
  end
  </code></pre>
  <p>If you want to get all the PDB entries you should use the mmCIF format, though it does lead to larger file sizes.</p>
  
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2021-11-08T12:11:57.843Z
Answers.LastActivityDate: 2021-11-08T12:11:57.843Z
Answers.UserDisplayName: jgreener
Answers.Body: <p>my attempt working as of the 17th January 2023 using RCSB PDB Search API:</p>
  <pre><code>import json
  
  # import urllib.parse
  
  import requests
  
  import sys
  
  # data = {&quot;query&quot;:{&quot;type&quot;:&quot;terminal&quot;,&quot;service&quot;:&quot;text&quot;,&quot;parameters&quot;:{&quot;attribute&quot;:&quot;rcsb_entry_container_identifiers.entry_id&quot;,
              # &quot;operator&quot;:&quot;exists&quot;,&quot;negation&quot;:False}},&quot;return_type&quot;:&quot;entry&quot;}
  
  data = {&quot;query&quot;:{&quot;type&quot;:&quot;terminal&quot;,&quot;service&quot;:&quot;text&quot;,&quot;parameters&quot;:{&quot;attribute&quot;:&quot;rcsb_entry_container_identifiers.entry_id&quot;,
               &quot;operator&quot;:&quot;exists&quot;,&quot;negation&quot;:False}},&quot;request_options&quot;:{&quot;return_all_hits&quot;:True},&quot;return_type&quot;:&quot;entry&quot;}
  
  data = json.dumps(data)
  
  
  headers = {'content-type': 'application/json'}
  url = 'https://search.rcsb.org/rcsbsearch/v2/query?json='
  
  params = {}
  
  risposta = requests.post(url, params=params, data = data, headers=headers)
  
  # print(risposta.status_code,'\n', risposta.content.decode())
  
  rispo = json.loads(risposta.content.decode())
  
  # print(rispo[&quot;result_set&quot;], type(rispo[&quot;result_set&quot;]), len(rispo[&quot;result_set&quot;]))
  
  
  id_list = []
  for i in rispo[&quot;result_set&quot;] :
      
      id_list.append(i['identifier'])
  
  
  print(id_list, type(id_list) , 'lenght : ', len(id_list), 'size bytes  : ' ,sys.getsizeof(id_list))
  </code></pre>
  <p>you get a huge list :</p>
  <pre><code>.............................
  '2V0H', '2V0G', '2V0F', '2V0E', '2V0D', '370D', '2ULL', '2UKD', '2UGI', '2UDP', '2UCZ', '2UBP', '2UAG', '2TYS', '369D', '368D', '367D', '366D', '2TUN', '2TSS', '2TSR', '2TSY', '365D', '2TRT', '2TRS', '2TS1', '2TRY', '2TRX', '364D', '2TSC', '2TSB', '2TSA', '363D', '2TRC', '2TRA', '2TRM', '2TRH'] 
  
  &lt;class 'list'&gt; lenght :  200069 size bytes  :  1671784
  </code></pre>
  <p>Using the requests GET endpoints :</p>
  <pre><code>import requests
  
  import sys
  
  
  url = 'https://search.rcsb.org/rcsbsearch/v2/query?json=%7B%22query%22%3A+%7B%22type%22%3A+%22terminal%22%2C+%22service%22%3A+%22text%22%2C+%22parameters%22%3A+%7B%22attribute%22%3A+%22rcsb_entry_container_identifiers.entry_id%22%2C+%22operator%22%3A+%22exists%22%2C+%22negation%22%3A+false%7D%7D%2C+%22request_options%22%3A+%7B%22return_all_hits%22%3A+true%7D%2C+%22return_type%22%3A+%22entry%22%7D'
  
  
  risposta = requests.get(url)
  
  
  print('+++++++++++++++++++++++++++++')
  
  rispo = risposta.json()
  
  id_list = []
  for i in rispo[&quot;result_set&quot;] :
      
      id_list.append(i['identifier'])
  
  print(id_list, type(id_list) , 'lenght : ', len(id_list), 'size bytes  : ' ,sys.getsizeof(id_list))
  
  print('--------------------------------------')
  
  print(risposta.status_code)
  
  print('\n\n',risposta.url)
  </code></pre>
  <p>getting as of the 19th January 2023 --&gt; 200375 entries</p>
  
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2023-01-17T00:36:54.427Z
Answers.LastActivityDate: 2023-01-19T15:17:13.560Z
Answers.LastEditDate: 2023-01-19T15:17:13.560Z
Answers.UserDisplayName: pippo1980
Answers.Body: <p>The page <a href="https://www.wwpdb.org/ftp/pdb-ftp-sites" rel="noreferrer">https://www.wwpdb.org/ftp/pdb-ftp-sites</a> gives the addresses of interest. PDB has an FTP site downloadable via rsync and the latter has a dry run mode.</p>
  <pre><code>rsync --port=33444 -rlpt -v -z --dry-run --stats rsync.rcsb.org::ftp_data/structures/divided/pdb/
  </code></pre>
  <p>You get a bunch of folders names and the following:</p>
  <pre><code>Number of files: 182479
  Number of files transferred: 0
  Total file size: 38382243431 bytes
  Total transferred file size: 0 bytes
  Literal data: 0 bytes
  Matched data: 0 bytes
  File list size: 3777395
  File list generation time: 52.909 seconds
  File list transfer time: 0.000 seconds
  Total bytes sent: 16
  Total bytes received: 3777651
  
  sent 16 bytes  received 3777651 bytes  66861.36 bytes/sec
  total size is 38382243431  speedup is 10160.30
  </code></pre>
  <p>So there are 182,479 *.ent.gz in various folders. The total is 38,382,243,431 bytes, which is 38 GB, so not too much. In Python it is easy to iterate across gzip files with the <a href="https://docs.python.org/3/library/gzip.html" rel="noreferrer">gzip module</a>, so uncompressing them is generally a thing to avoid as it will bloat up greatly. The problem arise when dealing with the density maps, which do take up a lot of space...</p>
  <h2>Download without saving</h2>
  <p>The PDBe is easier to for downloading things than the RCSB PDB and the API are much more comprehensive.</p>
  <p>Given a list of PDB codes one could</p>
  <pre><code>pdbcode : str = ''
  
  import requests
  r = requests.get(f'https://www.ebi.ac.uk/pdbe/entry-files/download/pdb{pdbcode.lower()}.ent', stream=True)
  assert r.status_code == 200, f'something ({r.status_code} is wrong: {r.text}'
  pdb_code = r.text
  </code></pre>
  
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2021-11-04T08:25:50.953Z
Answers.Text: There is >180,000 PDB entries, see https://www.rcsb.org/stats/growth/growth-released-structures
Answers.UserDisplayName: marcin
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2021-11-04T08:51:55.680Z
Answers.Text: Thanks for pointing that out! I thought that was suspicious. I forgot the recursive flag. Now the output is 38 GB, which is more reasonable. I don't have a folder to check in PDB format, but I thought it was a lot less...
Answers.UserDisplayName: Matteo Ferla
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2021-11-04T10:27:26.673Z
Answers.Text: I'll only add that there are entries (currently, 2375) that have coordinates in the mmCIF format (which is the primary format) and not in the PDB format. MmCIF files are somewhat bigger, the compressed archive is 52GB.
Answers.UserDisplayName: marcin
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2023-01-17T00:12:45.660Z
Answers.Text: is rsync a Python builtin ??
Answers.UserDisplayName: pippo1980
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2023-01-17T11:20:15.920Z
Answers.Text: It's a classic Unix command
Answers.UserDisplayName: Matteo Ferla
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2021-11-03T14:49:23.830Z
Answers.LastActivityDate: 2021-11-04T08:47:33.540Z
Answers.LastEditDate: 2021-11-04T08:47:33.540Z
Answers.UserDisplayName: Matteo Ferla
Body: <p>What would be the total size of all PDB files in RCSB?</p>
  <p>Can I traverse/enumerate them one at a time using a Python script without downloading all of them to my local disk?</p>
  <p>If YES, what would be the Python script like?</p>
  
Tags: <python><pdb><biopython>
Title: Can I traverse/enumerate PDB one at a time using a Python script without downloading all of them to my local disk?
ThreadText: # Can I traverse/enumerate PDB one at a time using a Python script without downloading all of them to my local disk?
  
  (Asked by: username_0 on 2021-11-03)
  
  What would be the total size of all PDB files in RCSB?
  
  Can I traverse/enumerate them one at a time using a Python script without downloading all of them to my local disk?
  
  If YES, what would be the Python script like?
  
  **Tags:** python, pdb, biopython
  
  
  ---
  
  ## 3 answers
  
  ### Answer by username_3 on 2021-11-03. Score: 6 | Accepted answer
  
  The page <https://www.wwpdb.org/ftp/pdb-ftp-sites> gives the addresses of interest. PDB has an FTP site downloadable via rsync and the latter has a dry run mode.
  
  ```
  rsync --port=33444 -rlpt -v -z --dry-run --stats rsync.rcsb.org::ftp_data/structures/divided/pdb/
  
  ```
  
  You get a bunch of folders names and the following:
  
  ```
  Number of files: 182479
  Number of files transferred: 0
  Total file size: 38382243431 bytes
  Total transferred file size: 0 bytes
  Literal data: 0 bytes
  Matched data: 0 bytes
  File list size: 3777395
  File list generation time: 52.909 seconds
  File list transfer time: 0.000 seconds
  Total bytes sent: 16
  Total bytes received: 3777651
  
  sent 16 bytes  received 3777651 bytes  66861.36 bytes/sec
  total size is 38382243431  speedup is 10160.30
  
  ```
  
  So there are 182,479 \*.ent.gz in various folders. The total is 38,382,243,431 bytes, which is 38 GB, so not too much. In Python it is easy to iterate across gzip files with the [gzip module](https://docs.python.org/3/library/gzip.html), so uncompressing them is generally a thing to avoid as it will bloat up greatly. The problem arise when dealing with the density maps, which do take up a lot of space...
  
  ## Download without saving
  
  The PDBe is easier to for downloading things than the RCSB PDB and the API are much more comprehensive.
  
  Given a list of PDB codes one could
  
  ```
  pdbcode : str = ''
  
  import requests
  r = requests.get(f'https://www.ebi.ac.uk/pdbe/entry-files/download/pdb{pdbcode.lower()}.ent', stream=True)
  assert r.status_code == 200, f'something ({r.status_code} is wrong: {r.text}'
  pdb_code = r.text
  
  ```
  
  > **Comments:**
  > **username_4**: There is >180,000 PDB entries, see https://www.rcsb.org/stats/growth/growth-released-structures  
  > **username_3**: Thanks for pointing that out! I thought that was suspicious. I forgot the recursive flag. Now the output is 38 GB, which is more reasonable. I don't have a folder to check in PDB format, but I thought it was a lot less...  
  > **username_4**: I'll only add that there are entries (currently, 2375) that have coordinates in the mmCIF format (which is the primary format) and not in the PDB format. MmCIF files are somewhat bigger, the compressed archive is 52GB.  
  > **username_2**: is rsync a Python builtin ??  
  > **username_3**: It's a classic Unix command  
  
  ---
  
  ### Answer by username_1 on 2021-11-08. Score: 0
  
  Not Python, but you can do this with [BioStructures.jl](https://biojulia.net/BioStructures.jl/stable) in Julia:
  
  ```
  using BioStructures
  pdblist = pdbentrylist()
  for pdbid in pdblist
      # Download a file and delete it after use
      downloadpdb(pdbid, format=MMCIF) do fp
          s = read(fp, MMCIF)
          # Do something with the file, e.g. print the cysteine fraction
          nres = countresidues(s, standardselector)
          if nres > 0
              frac = countresidues(s, standardselector, x -> resname(x) == "CYS") / nres
              println(pdbid, "  ", round(frac; digits=2))
          end
      end
  end
  
  ```
  
  If you want to get all the PDB entries you should use the mmCIF format, though it does lead to larger file sizes.
  
  
  ---
  
  ### Answer by username_2 on 2023-01-17. Score: 0
  
  my attempt working as of the 17th January 2023 using RCSB PDB Search API:
  
  ```
  import json
  
  # import urllib.parse
  
  import requests
  
  import sys
  
  # data = {"query":{"type":"terminal","service":"text","parameters":{"attribute":"rcsb_entry_container_identifiers.entry_id",
              # "operator":"exists","negation":False}},"return_type":"entry"}
  
  data = {"query":{"type":"terminal","service":"text","parameters":{"attribute":"rcsb_entry_container_identifiers.entry_id",
               "operator":"exists","negation":False}},"request_options":{"return_all_hits":True},"return_type":"entry"}
  
  data = json.dumps(data)
  
  headers = {'content-type': 'application/json'}
  url = 'https://search.rcsb.org/rcsbsearch/v2/query?json='
  
  params = {}
  
  risposta = requests.post(url, params=params, data = data, headers=headers)
  
  # print(risposta.status_code,'\n', risposta.content.decode())
  
  rispo = json.loads(risposta.content.decode())
  
  # print(rispo["result_set"], type(rispo["result_set"]), len(rispo["result_set"]))
  
  id_list = []
  for i in rispo["result_set"] :
      
      id_list.append(i['identifier'])
  
  print(id_list, type(id_list) , 'lenght : ', len(id_list), 'size bytes  : ' ,sys.getsizeof(id_list))
  
  ```
  
  you get a huge list :
  
  ```
  .............................
  '2V0H', '2V0G', '2V0F', '2V0E', '2V0D', '370D', '2ULL', '2UKD', '2UGI', '2UDP', '2UCZ', '2UBP', '2UAG', '2TYS', '369D', '368D', '367D', '366D', '2TUN', '2TSS', '2TSR', '2TSY', '365D', '2TRT', '2TRS', '2TS1', '2TRY', '2TRX', '364D', '2TSC', '2TSB', '2TSA', '363D', '2TRC', '2TRA', '2TRM', '2TRH'] 
  
  <class 'list'> lenght :  200069 size bytes  :  1671784
  
  ```
  
  Using the requests GET endpoints :
  
  ```
  import requests
  
  import sys
  
  url = 'https://search.rcsb.org/rcsbsearch/v2/query?json=%7B%22query%22%3A+%7B%22type%22%3A+%22terminal%22%2C+%22service%22%3A+%22text%22%2C+%22parameters%22%3A+%7B%22attribute%22%3A+%22rcsb_entry_container_identifiers.entry_id%22%2C+%22operator%22%3A+%22exists%22%2C+%22negation%22%3A+false%7D%7D%2C+%22request_options%22%3A+%7B%22return_all_hits%22%3A+true%7D%2C+%22return_type%22%3A+%22entry%22%7D'
  
  risposta = requests.get(url)
  
  print('+++++++++++++++++++++++++++++')
  
  rispo = risposta.json()
  
  id_list = []
  for i in rispo["result_set"] :
      
      id_list.append(i['identifier'])
  
  print(id_list, type(id_list) , 'lenght : ', len(id_list), 'size bytes  : ' ,sys.getsizeof(id_list))
  
  print('--------------------------------------')
  
  print(risposta.status_code)
  
  print('\n\n',risposta.url)
  
  ```
  
  getting as of the 19th January 2023 --> 200375 entries
  
  
  ---
  
  