 ;;     (assoc merged :error-paths (clojure.set/union (:error-paths total)
 ;;                                                   (:error-paths store)))))
 
-(defn- deref-stats
-  [stats]
-  {:processed (deref (:processed stats))
-   :error-paths (deref (:error-paths stats))})
-
 (defn- show-stats
   "Show metric store stats."
   [title data-stats store-stats]

    (when-not valid?
      (if-not @error-reported?
        (do
          (swap! error-paths conj path)
          (wlog/error err)
          (swap! error-reported? (fn [_] true)))
        (log/debug err)))
    valid?))

(defn- validate-points
  "Validate points."
  [mstore options tenant rollup period retention path series]
  (let [series (whisper/sort-series series)
        from (first (first series))
        to (first (last series))
        min-ttl (:min-ttl options default-min-ttl)
        mstore-series (mstore/fetch-series mstore tenant rollup period path
                                           from to)
        error-reported? (atom false)
        validators [[#(not= %2 nil) "Point not found"]
                    [#(sequential? %2) "Value is not an array"]
                    [#(= (count %2) 1) "Array size is not equal to 1"]
                    [#(= %1 (first %2)) "Values are not equal"]]
        error-paths (:error-paths (:data-mstore-stats options))]
    (doseq [point series]
      (let [time (first point)
            w-value (last point)
            ttl (calc-ttl time retention)]
        (when (> ttl min-ttl)
          (log-point "Validating point" rollup period path time w-value ttl)
          (let [s-value (get mstore-series time)]
            (every? #(validate-value rollup period path time w-value s-value
                                     (first %) (second %) error-reported?
                                     error-paths)
                    validators)))))))

(defn- process-archive
  "Process an archive."
  [ra-file file archive mstore tenant path from to retention options points-fn]
  (let [rollup (:seconds-per-point archive)
        points (:points archive)
        period (if retention (/ retention rollup) points)
        retention (if retention retention (* rollup points))
        data (whisper/fetch-archive-seq-ra ra-file file archive from to)
        series (whisper/remove-nulls (:series data))]
    (when points-fn
      (points-fn mstore options tenant rollup period retention path series))))

(defn- migrate-path
  "Migrate a path."
  [pstore tenant path options]
  (let [run (:run options false)]
    (when run
      (pstore/insert pstore tenant path))))

(defn- validate-path
  "Validate a path."
  [pstore tenant path options]
  (let [error-paths (:error-paths (:data-pstore-stats options))]
    (when-not (pstore/exist? pstore tenant path)
      (swap! error-paths conj path)
      (wlog/error "Path not found in the path store: " path))))

(defn- process-file
  "Process a file."
  [file dir mstore pstore tenant from to options path-fn points-fn]
  (let [path (whisper/file-to-name file dir)
        mstore-processed (:processed (:data-mstore-stats options))
        pstore-processed (:processed (:data-pstore-stats options))]
    (when (and pstore path-fn)
      (swap! pstore-processed inc)
      (path-fn pstore tenant path options))
    (when mstore
      (let [ra-file (RandomAccessFile. file "r")
            rollups (:rollups options)]

                  (let [retention (get rollups seconds-per-point
                                       (:retention archive))]
                    (process-archive ra-file file archive mstore tenant path
                                     from to retention options processor))))))
          (finally
            (.close ra-file)))))))

