 
             next-pg-content (rate-limited-download next-pg-url)]
         (write-content start-epoch next-pg-content)
-        (recur start-epoch
-               next-pg-content)))))
+        (when-not (stop-iteration? start-epoch (get next-pg-content "response"))
+          (recur start-epoch
+                 next-pg-content))))))
 
 (defn threads-since
   [start-epoch]
   (let [request-url (create-url start-epoch)
         first-page  (rate-limited-download request-url)]
     (write-content start-epoch first-page)
-    (discover-iteration start-epoch first-page-content)))
+    (discover-iteration start-epoch first-page)))

(ns clueweb-disqus.core
  (:require [cheshire.core :refer :all]
            [clj-http.client :as client]
            [clj-time.core :as t]
            [clojure.java.io :as io])
  (:use [clojure.pprint :only [pprint]]))


            (swap! requests-in-session inc)
            (-> a-url client/get :body parse-string))

        (and (< (t/in-minutes (t/interval @session-start-time (t/now)))
                60)
             (> 1000 @requests-in-session))
        (do (swap! requests-in-session inc)
            (-> a-url client/get :body parse-string))

        (and (< (t/in-minutes (t/interval @session-start-time (t/now)))
                60)
             (<= 1000 @requests-in-session))
        (let [time-to-sleep (* 1000
                               (-
                                3600
                                (t/in-secs
                                 (t/interval @session-start-time
                                             (t/now)))))]
          (do (Thread/sleep time-to-sleep) ; sleep till the end of the
                                          ; hour
                                          ; and then restart the session
              (swap! session-start-time (fn [x] (t/now)))
