             (not (.isDone @download-supervisor))))))
 
 (defn- update-domains
-  "TODO: Always cleanup after update?"
   [download-supervisor domains-info edb-config rw-lock]
   (let [{max-kbs :max-online-download-rate-kb-s
          local-dir :local-dir} edb-config

        (try (purge-unused-domains! domain-seq local-dir)
             (load-cached-domains! edb-config rw-lock domains-info)
             (load-and-sync-status! edb-config rw-lock domains-info)
             (cleanup-domains! domain-seq local-dir)
             (log-message "Finished loading all updated domains from remote.")
             (catch Throwable t
               (log-error t "Error when syncing data.")

        (let [{:keys [finished-loaders shard-states]} state]
          (swap! finished-loaders + (count (shard-states domain))))))))

(defn cleanup-domain!
  [domain-path]
  "Destroys all but the most recent version in the versioned store
  located at `domain-path`."
  (doto (DomainStore. (local-filesystem) domain-path)
    (.cleanup 1)))

(defn cleanup-domains!
  "Destroys every old version for each domain in `domains` located
  inside of `local-dir`. (Each domain directory contains a versioned
  store, capable of being 'cleaned up' in this fashion.)

  If any cleanup operation throws an error, `cleanup-domains!` will
  try to operate on the rest of the domains, and throw a single error
  on completion."
  [domains local-dir]
  (let [error (atom nil)]
    (dofor [domain domains :let [domain-path (str (path local-dir domain))]]
           (try (log-message "Purging old versions of domain: " domain)
                (cleanup-domain! domain-path)
                (catch Throwable t
                  (log-error t "Error when purging old versions of domain: " domain)
                  (reset! error t))))
    (when-let [e @error]
      (throw e))))

(defn load-and-sync-status!
  "TODO: Only note success for non-failed domains. check thrift status."
  [edb-config rw-lock domains-info & {:keys [state]}]
  (let [{remote-path-map :domains
         local-dir :local-dir} edb-config
         status (thrift/ready-status :loading? true)
         domain-names (keys domains-info)
         loaders (p-dofor [[domain info] domains-info
                           :let [remote-path (get remote-path-map domain)
                                 loader-state (or state (mk-loader-state
                                                         {domain (domain/host-shards info)}))]]
                          (try (domain/set-domain-status! info status)
                               (when-let [new-data (use-cache-or-update domain
                                                                        remote-path
                                                                        edb-config
                                                                        loader-state)]  
                                 (domain/set-domain-data! rw-lock domain info new-data))
                               (domain/set-domain-status! info (thrift/ready-status))
                               (catch Throwable t
                                 (log-error t "Error when loading domain " domain)
                                 (domain/set-domain-status! info (thrift/failed-status t)))))]
    (with-ret loaders
      (try (log-message "Removing all old versions of updated domains!")
           (cleanup-domains! domain-names local-dir)
           (catch Throwable t
             (log-error t "Error when cleaning old versions.")))
      (log-message "Successfully loaded domains: " (s/join ", " domain-names)))))

(defn load-cached-domains!
  [edb-config rw-lock domains-info]
  (let [{:keys [hdfs-conf local-dir local-db-conf]} edb-config
        remote-path-map (:domains edb-config)
        status (thrift/loading-status)
        domain-names (keys domains-info)
        loaders (p-dofor [[domain info] domains-info]
                         (try (domain/set-domain-status! info status)
                              (let [remote-path  (get remote-path-map domain)
                                    local-domain-root (str (path local-dir domain))
                                    local-vs (mk-local-vs (filesystem hdfs-conf)
                                                          remote-path
